Merge branch 'master' into fix/ING-722-remove-sqlparse-linege-deps

sagar-salvi-apptware · Dec 10, 2024 · 70d2606 · 70d2606
2 parents 135561c + 638a0e3
commit 70d2606
Show file tree

Hide file tree

Showing 130 changed files with 118,675 additions and 1,252 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -34,29 +34,21 @@ jobs:
         include:
           # Note: this should be kept in sync with tox.ini.
           - python-version: "3.8"
-            extra_pip_requirements: "apache-airflow~=2.1.4"
-            extra_pip_extras: plugin-v1
-          - python-version: "3.8"
-            extra_pip_requirements: "apache-airflow~=2.2.4"
-            extra_pip_extras: plugin-v1
+            extra_pip_requirements: "apache-airflow~=2.3.4"
+            extra_pip_extras: test-airflow23
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.4.3"
-            extra_pip_extras: plugin-v2,test-airflow24
+            extra_pip_extras: test-airflow24
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.10"
             extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.11"
             extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
-            extra_pip_extras: plugin-v2
           - python-version: "3.11"
-            extra_pip_requirements: "apache-airflow~=2.10.2 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.11.txt"
-            extra_pip_extras: plugin-v2
+            extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
       fail-fast: false
     steps:
       - name: Set up JDK 17

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -75,6 +75,8 @@ jobs:
           path: |
             ~/.cache/uv
           key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
+      - name: Install dependencies
+        run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -29,24 +29,24 @@ jobs:
                 "swaroopjagadish",
                 "treff7es",
                 "yoonhyejin",
-                "eboneil",
                 "gabe-lyons",
                 "hsheth2",
                 "jjoyce0510",
                 "maggiehays",
                 "pedro93",
                 "RyanHolstien",
                 "sakethvarma397",
-                "Kunal-kankriya",
                 "purnimagarg1",
-                "dushayntAW",
                 "sagar-salvi-apptware",
                 "kushagra-apptware",
                 "Salman-Apptware",
                 "mayurinehate",
                 "noggi",
                 "skrydal",
-                "kevinkarchacryl"
+                "kevinkarchacryl",
+                "sgomezvillamor",
+                "acrylJonny",
+                "chakru-r"
               ]'), 
               github.actor
             ) 

diff --git a/...react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx b/...react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx
@@ -5,6 +5,8 @@ import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/searc
 import { Container, Entity, EntityType } from '../../../../../../../types.generated';
 import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener';
 import { useEntityRegistry } from '../../../../../../useEntityRegistry';
+import { getParentEntities } from '../../../../../../search/filters/utils';
+import ParentEntities from '../../../../../../search/filters/ParentEntities';
 
 type Props = {
     onCloseModal: () => void;
@@ -26,14 +28,18 @@ const StyleTag = styled(Tag)`
     align-items: center;
 `;
 
-const PreviewImage = styled.img`
+export const PreviewImage = styled.img`
     max-height: 18px;
     width: auto;
     object-fit: contain;
     background-color: transparent;
     margin-right: 4px;
 `;
 
+export const ParentWrapper = styled.div`
+    max-width: 400px;
+`;
+
 export const ContainerSelectModal = ({ onCloseModal, defaultValues, onOkOverride, titleOverride }: Props) => {
     const [containerSearch, { data: platforSearchData }] = useGetSearchResultsLazyQuery();
     const entityRegistry = useEntityRegistry();
@@ -65,10 +71,16 @@ export const ContainerSelectModal = ({ onCloseModal, defaultValues, onOkOverride
     // Renders a search result in the select dropdown.
     const renderSearchResult = (entity: Container) => {
         const displayName = entityRegistry.getDisplayName(EntityType.Container, entity);
+        const parentEntities: Entity[] = getParentEntities(entity as Entity) || [];
 
         const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName;
         return (
             <Tooltip title={displayName}>
+                {parentEntities.length > 0 && (
+                    <ParentWrapper>
+                        <ParentEntities parentEntities={parentEntities} />
+                    </ParentWrapper>
+                )}
                 <PreviewImage src={entity.platform?.properties?.logoUrl || undefined} alt={entity.properties?.name} />
                 <span>{truncatedDisplayName}</span>
             </Tooltip>

diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx
@@ -16,6 +16,7 @@ const StyledCollapse = styled(Collapse)<{ color: string }>`
         .ant-collapse-header {
             display: flex;
             align-items: center;
+            overflow: auto;
         }
 
         .ant-collapse-item {

diff --git a/datahub-web-react/src/app/search/SearchFilterLabel.tsx b/datahub-web-react/src/app/search/SearchFilterLabel.tsx
@@ -23,6 +23,9 @@ import CustomAvatar from '../shared/avatar/CustomAvatar';
 import { IconStyleType } from '../entity/Entity';
 import { formatNumber } from '../shared/formatNumber';
 import useGetBrowseV2LabelOverride from './filters/useGetBrowseV2LabelOverride';
+import { getParentEntities } from './filters/utils';
+import { ParentWrapper } from '../entity/shared/containers/profile/sidebar/Container/ContainerSelectModal';
+import ParentEntities from './filters/ParentEntities';
 
 type Props = {
     field: string;
@@ -157,11 +160,17 @@ export const SearchFilterLabel = ({ field, value, entity, count, hideCount }: Pr
     if (entity?.type === EntityType.Container) {
         const container = entity as Container;
         const displayName = entityRegistry.getDisplayName(EntityType.Container, container);
+        const parentEntities: Entity[] = getParentEntities(container as Entity) || [];
         const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName;
         return (
             <Tooltip title={displayName}>
                 {!!container.platform?.properties?.logoUrl && (
-                    <PreviewImage src={container.platform?.properties?.logoUrl} alt={container.properties?.name} />
+                    <>
+                        <ParentWrapper style={{ width: '200px' }}>
+                            <ParentEntities parentEntities={parentEntities} />
+                        </ParentWrapper>
+                        <PreviewImage src={container.platform?.properties?.logoUrl} alt={container.properties?.name} />
+                    </>
                 )}
                 <span>
                     {truncatedDisplayName}

diff --git a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx
@@ -4,7 +4,12 @@ import { FacetFilterInput, FacetMetadata } from '../../types.generated';
 import { FilterScenarioType } from './filters/render/types';
 import { useFilterRendererRegistry } from './filters/render/useFilterRenderer';
 import { SimpleSearchFilter } from './SimpleSearchFilter';
-import { ENTITY_FILTER_NAME, ENTITY_INDEX_FILTER_NAME, LEGACY_ENTITY_FILTER_NAME } from './utils/constants';
+import {
+    DEGREE_FILTER_NAME,
+    ENTITY_FILTER_NAME,
+    ENTITY_INDEX_FILTER_NAME,
+    LEGACY_ENTITY_FILTER_NAME,
+} from './utils/constants';
 
 const TOP_FILTERS = ['degree', ENTITY_FILTER_NAME, 'platform', 'tags', 'glossaryTerms', 'domains', 'owners'];
 
@@ -43,6 +48,15 @@ export const SimpleSearchFilters = ({ facets, selectedFilters, onFilterSelect, l
                           : filter,
                   )
                   .filter((filter) => filter.field !== field || !(filter.values?.length === 0));
+
+        // Do not let user unselect all degree filters
+        if (field === DEGREE_FILTER_NAME && !selected) {
+            const hasDegreeFilter = newFilters.find((filter) => filter.field === DEGREE_FILTER_NAME);
+            if (!hasDegreeFilter) {
+                return;
+            }
+        }
+
         setCachedProps({ ...cachedProps, selectedFilters: newFilters });
         onFilterSelect(newFilters);
     };

diff --git a/datahub-web-react/src/app/search/filters/FilterOption.tsx b/datahub-web-react/src/app/search/filters/FilterOption.tsx
@@ -8,6 +8,7 @@ import { generateColor } from '../../entity/shared/components/styled/StyledTag';
 import { ANTD_GRAY } from '../../entity/shared/constants';
 import { useEntityRegistry } from '../../useEntityRegistry';
 import {
+    CONTAINER_FILTER_NAME,
     ENTITY_SUB_TYPE_FILTER_NAME,
     MAX_COUNT_VAL,
     PLATFORM_FILTER_NAME,
@@ -125,7 +126,7 @@ export default function FilterOption({
     const { field, value, count, entity } = filterOption;
     const entityRegistry = useEntityRegistry();
     const { icon, label } = getFilterIconAndLabel(field, value, entityRegistry, entity || null, 14);
-    const shouldShowIcon = field === PLATFORM_FILTER_NAME && icon !== null;
+    const shouldShowIcon = (field === PLATFORM_FILTER_NAME || field === CONTAINER_FILTER_NAME) && icon !== null;
     const shouldShowTagColor = field === TAGS_FILTER_NAME && entity?.type === EntityType.Tag;
     const isSubTypeFilter = field === TYPE_NAMES_FILTER_NAME;
     const parentEntities: Entity[] = getParentEntities(entity as Entity) || [];

diff --git a/datahub-web-react/src/app/search/filters/utils.tsx b/datahub-web-react/src/app/search/filters/utils.tsx
@@ -20,6 +20,7 @@ import {
     FacetFilterInput,
     FacetMetadata,
     GlossaryTerm,
+    Container,
 } from '../../../types.generated';
 import { IconStyleType } from '../../entity/Entity';
 import {
@@ -186,6 +187,15 @@ export function getFilterIconAndLabel(
             entityRegistry.getIcon(EntityType.DataPlatform, size || 12, IconStyleType.ACCENT, ANTD_GRAY[9])
         );
         label = filterEntity ? entityRegistry.getDisplayName(EntityType.DataPlatform, filterEntity) : filterValue;
+    } else if (filterField === CONTAINER_FILTER_NAME) {
+        // Scenario where the filter entity exists and filterField is container
+        const logoUrl = (filterEntity as Container)?.platform?.properties?.logoUrl;
+        icon = logoUrl ? (
+            <PlatformIcon src={logoUrl} size={size} />
+        ) : (
+            entityRegistry.getIcon(EntityType.DataPlatform, size || 12, IconStyleType.ACCENT, ANTD_GRAY[9])
+        );
+        label = entityRegistry.getDisplayName(EntityType.Container, filterEntity);
     } else if (filterField === BROWSE_PATH_V2_FILTER_NAME) {
         icon = <FolderFilled size={size} color="black" />;
         label = getLastBrowseEntryFromFilterValue(filterValue);
@@ -196,6 +206,7 @@ export function getFilterIconAndLabel(
             filterEntity,
             size,
         );
+
         icon = newIcon;
         label = newLabel;
     } else {
@@ -344,6 +355,9 @@ export function getParentEntities(entity: Entity): Entity[] | null {
     if (entity.type === EntityType.Domain) {
         return (entity as Domain).parentDomains?.domains || [];
     }
+    if (entity.type === EntityType.Container) {
+        return (entity as Container).parentContainers?.containers || [];
+    }
     return null;
 }
 

diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql
@@ -1010,6 +1010,7 @@ fragment entityContainer on Container {
 
 fragment parentContainerFields on Container {
     urn
+    type
     properties {
         name
     }

diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql
@@ -910,6 +910,9 @@ fragment facetFields on FacetMetadata {
                 properties {
                     name
                 }
+                parentContainers {
+                    ...parentContainersFields
+                }
             }
             ... on CorpUser {
                 username

diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
@@ -5043,9 +5043,9 @@ [email protected]:
     tslib "^2.4.0"
 
 cross-spawn@^7.0.2:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -149,6 +149,25 @@ module.exports = {
           type: "doc",
           id: "docs/glossary/business-glossary",
         },
+        {
+          label: "Compliance Forms",
+          type: "category",
+          collapsed: true,
+          items: [
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/compliance-forms/overview",
+            },
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/compliance-forms/create-a-form",
+            },
+            {
+              type: "doc",
+              id: "docs/features/feature-guides/compliance-forms/complete-a-form",
+            },
+          ],
+        },
         {
           label: "Data Contract",
           type: "doc",
@@ -164,7 +183,6 @@ module.exports = {
           type: "doc",
           id: "docs/features/dataset-usage-and-query-history",
         },
-        "docs/features/feature-guides/documentation-forms",
         {
           label: "Domains",
           type: "doc",

diff --git a/docs/api/tutorials/forms.md b/docs/api/tutorials/forms.md
@@ -1,13 +1,15 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Documentation Forms
+# Compliance Forms
 
-## Why Would You Use Documentation Forms?
+## Why Would You Use Compliance Forms?
 
-Documentation Forms are a way for end-users to fill out all mandatory attributes associated with a data asset. The form will be dynamically generated based on the definitions provided by administrators and stewards and matching rules.
+**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach.
 
-Learn more about forms in the [Documentation Forms Feature Guide](../../../docs/features/feature-guides/documentation-forms.md).
+With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts.
+
+Learn more about forms in the [Compliance Forms Feature Guide](../../../docs/features/feature-guides/compliance-forms/overview.md).
 
 ### Goal Of This Guide
 This guide will show you how to
-Original file line number
+Diff line change
@@ Expand Up / @@ -1010,6 +1010,7 @@ fragment entityContainer on Container { @@
     fragment parentContainerFields on Container {
         urn
+        type
         properties {
             name
         }
@@ Expand Down @@