diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index b3728aa6764903..732a782139b616 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -645,7 +645,7 @@ type Mutation {
     updateERModelRelationship(
         "The urn of the ERModelRelationship to update"
         urn: String!,
-        "Input required to updat an existing DataHub View"
+        "Input required to update an existing DataHub View"
         input: ERModelRelationshipUpdateInput!): Boolean
 
     """
@@ -795,7 +795,7 @@ type Mutation {
     updateView(
       "The urn of the View to update"
       urn: String!,
-      "Input required to updat an existing DataHub View"
+      "Input required to update an existing DataHub View"
       input: UpdateViewInput!): DataHubView
 
     """
diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java
index 68023a084bbd20..b471e366e906ff 100644
--- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java
+++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java
@@ -219,6 +219,6 @@ public void testMCPBatch() throws IOException {
         OP_CONTEXT
             .getObjectMapper()
             .readTree(
-                "{\"source\":{\"type\":\"datahub-gc\",\"config\":{\"cleanup_expired_tokens\":false,\"truncate_indices\":true,\"dataprocess_cleanup\":{\"retention_days\":10,\"delete_empty_data_jobs\":true,\"delete_empty_data_flows\":true,\"hard_delete_entities\":false,\"keep_last_n\":5},\"soft_deleted_entities_cleanup\":{\"retention_days\":10}}}}"));
+                "{\"source\":{\"type\":\"datahub-gc\",\"config\":{\"cleanup_expired_tokens\":false,\"truncate_indices\":true,\"dataprocess_cleanup\":{\"retention_days\":10,\"delete_empty_data_jobs\":true,\"delete_empty_data_flows\":true,\"hard_delete_entities\":false,\"keep_last_n\":5},\"soft_deleted_entities_cleanup\":{\"retention_days\":10},\"execution_request_cleanup\":{\"keep_history_min_count\":10,\"keep_history_max_count\":1000,\"keep_history_max_days\":30,\"batch_read_size\":100,\"enabled\":false}}}}"));
   }
 }
diff --git a/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml b/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
index d049a807ac1d88..233db06d61c3fc 100644
--- a/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
+++ b/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
@@ -23,7 +23,13 @@
               keep_last_n: {{dataprocess_cleanup.keep_last_n}}{{^dataprocess_cleanup.keep_last_n}}5{{/dataprocess_cleanup.keep_last_n}}
             soft_deleted_entities_cleanup:
               retention_days: {{soft_deleted_entities_cleanup.retention_days}}{{^soft_deleted_entities_cleanup.retention_days}}10{{/soft_deleted_entities_cleanup.retention_days}}
+            execution_request_cleanup:
+              keep_history_min_count: {{execution_request_cleanup.keep_history_min_count}}{{^execution_request_cleanup.keep_history_min_count}}10{{/execution_request_cleanup.keep_history_min_count}}
+              keep_history_max_count: {{execution_request_cleanup.keep_history_max_count}}{{^execution_request_cleanup.keep_history_max_count}}1000{{/execution_request_cleanup.keep_history_max_count}}
+              keep_history_max_days: {{execution_request_cleanup.keep_history_max_days}}{{^execution_request_cleanup.keep_history_max_days}}30{{/execution_request_cleanup.keep_history_max_days}}
+              batch_read_size: {{execution_request_cleanup.batch_read_size}}{{^execution_request_cleanup.batch_read_size}}100{{/execution_request_cleanup.batch_read_size}}
+              enabled: {{execution_request_cleanup.enabled}}{{^execution_request_cleanup.enabled}}false{{/execution_request_cleanup.enabled}}
       extraArgs: {}
       debugMode: false
       executorId: default
-  headers: {}
\ No newline at end of file
+  headers: {}
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
index fda8c9cda2d0da..663575f062093e 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import { message, Modal, Button, Form, Input, Typography, Select } from 'antd';
 import { useApolloClient } from '@apollo/client';
-import TextArea from 'antd/lib/input/TextArea';
+import styled from 'styled-components';
 import analytics, { EventType, EntityActionType } from '../../../../../analytics';
 import { useEntityData } from '../../../EntityContext';
 import { EntityType, IncidentSourceType, IncidentState, IncidentType } from '../../../../../../types.generated';
@@ -9,6 +9,12 @@ import { INCIDENT_DISPLAY_TYPES, PAGE_SIZE, addActiveIncidentToCache } from '../
 import { useRaiseIncidentMutation } from '../../../../../../graphql/mutations.generated';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
 import { useUserContext } from '../../../../../context/useUserContext';
+import { Editor } from '../../Documentation/components/editor/Editor';
+import { ANTD_GRAY } from '../../../constants';
+
+const StyledEditor = styled(Editor)`
+    border: 1px solid ${ANTD_GRAY[4.5]};
+`;
 
 type AddIncidentProps = {
     open: boolean;
@@ -112,6 +118,7 @@ export const AddIncidentModal = ({ open, onClose, refetch }: AddIncidentProps) =
                 open={open}
                 destroyOnClose
                 onCancel={handleClose}
+                width={600}
                 footer={[
                     <Button type="text" onClick={handleClose}>
                         Cancel
@@ -174,7 +181,17 @@ export const AddIncidentModal = ({ open, onClose, refetch }: AddIncidentProps) =
                             },
                         ]}
                     >
-                        <TextArea placeholder="Provide some additional details" />
+                        <StyledEditor
+                            doNotFocus
+                            className="add-incident-description"
+                            onKeyDown={(e) => {
+                                // Preventing the modal from closing when the Enter key is pressed
+                                if (e.key === 'Enter') {
+                                    e.preventDefault();
+                                    e.stopPropagation();
+                                }
+                            }}
+                        />
                     </Form.Item>
                 </Form>
             </Modal>
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
index dc123398cd2da4..3438003c2f17b3 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
@@ -14,6 +14,7 @@ import { useUpdateIncidentStatusMutation } from '../../../../../../graphql/mutat
 import { ResolveIncidentModal } from './ResolveIncidentModal';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
 import { MenuItemStyle } from '../../../../view/menu/item/styledComponent';
+import MarkdownViewer from '../../../components/legacy/MarkdownViewer';
 
 type Props = {
     incident: any;
@@ -227,7 +228,7 @@ export default function IncidentListItem({ incident, refetch }: Props) {
                             </TitleContainer>
                             <DescriptionContainer>
                                 <IncidentDescriptionLabel>Description</IncidentDescriptionLabel>
-                                <IncidentDescriptionText>{incident?.description}</IncidentDescriptionText>
+                                <MarkdownViewer source={incident?.description} />
                                 {incident.status.state === IncidentState.Resolved ? (
                                     <>
                                         <IncidentDescriptionLabel>Resolution Note</IncidentDescriptionLabel>
diff --git a/docs-website/adoptionStoriesIndexes.json b/docs-website/adoptionStoriesIndexes.json
index f7b78e6cab8b2a..618cd4bbb33342 100644
--- a/docs-website/adoptionStoriesIndexes.json
+++ b/docs-website/adoptionStoriesIndexes.json
@@ -20,6 +20,16 @@
       "category": "Financial & Fintech",
       "description": "<i>\"We found DataHub to provide excellent coverage for our needs. What we appreciate most about DataHub is <b>its powerful API platform.</b>\"<br /><br /><div style='color: gray;'>— Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA</div></i><br />"
     },
+    {
+      "name": "Apple",
+      "slug": "apple",
+      "link": "https://youtu.be/5eFZuzG4c-s?feature=shared",
+      "linkType": "video",
+      "tagline": "How Apple built a solid foundation for observability, governance, and data sharing with DataHub",
+      "category": "B2B & B2C",
+      "platform": "cloud",
+      "description": "Discover how DataHub provides a solid foundation for observability, governance, and data sharing, while we explore its role in managing AI and data metadata."
+    },
     {
       "name": "Notion",
       "slug": "notion",
diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
index 8c44b3c8b40bfa..31644f459ed731 100644
--- a/docs-website/docusaurus.config.js
+++ b/docs-website/docusaurus.config.js
@@ -66,9 +66,9 @@ module.exports = {
     //   },
     // }),
     announcementBar: {
-          id: "announcement-2",
+          id: "announcement-3",
           content:
-            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div><p>Join us at Metadata & AI Summit, Oct. 29 & 30!</p><a href="http://www.acryldata.io/conference?utm_source=datahub_web&utm_medium=metadata_ai_2024&utm_campaign=home_banner" target="_blank" class="button">Register<span> →</span></a></div>',
+            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><!--div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div--><p>Watch Metadata & AI Summit sessions on-demand.</p><a href="https://www.youtube.com/@DataHubProject/videos" target="_blank" class="button">Watch Now<span> →</span></a></div>',
           backgroundColor: "#111",
           textColor: "#ffffff",
           isCloseable: false,
diff --git a/docs-website/package.json b/docs-website/package.json
index e1130148864e5e..52f04e46c42aa2 100644
--- a/docs-website/package.json
+++ b/docs-website/package.json
@@ -35,6 +35,7 @@
     "@octokit/plugin-throttling": "^3.5.1",
     "@octokit/rest": "^18.6.2",
     "@radix-ui/react-visually-hidden": "^1.0.2",
+    "@servicebell/widget": "^0.1.6",
     "@supabase/supabase-js": "^2.33.1",
     "@swc/core": "^1.4.2",
     "antd": "^5.0.7",
@@ -47,6 +48,7 @@
     "markprompt": "^0.1.7",
     "react": "^18.2.0",
     "react-dom": "18.2.0",
+    "react-use-draggable-scroll": "^0.4.7",
     "sass": "^1.43.2",
     "swc-loader": "^0.2.6",
     "swiper": "^11.1.4",
diff --git a/docs-website/src/pages/_components/CaseStudy/case-study.module.scss b/docs-website/src/pages/_components/CaseStudy/case-study.module.scss
index e17f1f74656dc1..29cade9901c4d3 100644
--- a/docs-website/src/pages/_components/CaseStudy/case-study.module.scss
+++ b/docs-website/src/pages/_components/CaseStudy/case-study.module.scss
@@ -42,6 +42,9 @@
       flex: 1;
     }
     .cardLink {
+      user-select: none;
+      -webkit-user-drag: none;
+
       color: #000;
       
       &:hover {
@@ -88,6 +91,8 @@
           z-index: 10;
           filter: brightness(2);
           opacity: .9;
+          user-select: none;
+          -webkit-user-drag: none;
         }
         .cardImageBackground {
           position: absolute;
diff --git a/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js b/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js
index 3c840e4fab6eb2..4bb1b769b288aa 100644
--- a/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js
+++ b/docs-website/src/pages/_components/CaseStudy/caseStudyContent.js
@@ -9,6 +9,16 @@ const caseStudyData = [
     image: "https://datahubproject.io/img/logos/companies/netflix.png",
     link: "https://datahubproject.io/adoption-stories/#netflix",
   },
+  {
+    title: "A Solid Foundation For Data and AI",
+    description:
+      "How Apple built a solid foundation for observability, governance, and data sharing with DataHub.",
+    tag: "Technology",
+    backgroundImage:
+      "https://upload.wikimedia.org/wikipedia/commons/thumb/5/5a/Aerial_view_of_Apple_Park_dllu.jpg/2560px-Aerial_view_of_Apple_Park_dllu.jpg",
+    image: "/img/logos/companies/apple_text.png",
+    link: "https://datahubproject.io/adoption-stories/#apple",
+  },
   {
     title: "Scaling Data Governance",
     description:
diff --git a/docs-website/src/pages/_components/CaseStudy/index.js b/docs-website/src/pages/_components/CaseStudy/index.js
index 0362dec9a0d3ef..27766a8456a6fe 100644
--- a/docs-website/src/pages/_components/CaseStudy/index.js
+++ b/docs-website/src/pages/_components/CaseStudy/index.js
@@ -1,11 +1,15 @@
-import React from "react";
+import React, { useRef } from "react";
 import styles from "./case-study.module.scss";
 import Link from '@docusaurus/Link'
-import { Carousel } from "antd";
+import { useDraggable } from "react-use-draggable-scroll";
 import caseStudyData from "./caseStudyContent";
 
 
 const CaseStudy = () => {
+  const containerRef = useRef(null);
+
+  const { events } = useDraggable(containerRef);
+
   return (
     <div className={styles.container}>
       {/* Section-1 */}
@@ -15,16 +19,16 @@ const CaseStudy = () => {
           <p>Across finance, healthcare, e-commerce, and countless more.</p>
         </div>
 
-        <div className={styles.card_row}>
-          <div className={styles.card_row_wrapper} >
+        <div className={styles.card_row} {...events} ref={containerRef}>
+          <div className={styles.card_row_wrapper}>
             {caseStudyData.map((caseStudy) => (
               <div className={styles.card} key={caseStudy.link}>
-                <a className={styles.cardLink} href={caseStudy.link} style={caseStudy.backgroundImage ? null : {
+                <a draggable={false} className={styles.cardLink} href={caseStudy.link} style={caseStudy.backgroundImage ? null : {
                   opacity: .5
                 }}>
                   {caseStudy.tag ? <span className={styles.card_tag}>{caseStudy.tag}</span> : null}
                   <div className={styles.card_image} style={{ backgroundColor: caseStudy.backgroundImage ? null : '#eee' }}>
-                    <img src={caseStudy.image} alt="" />
+                    <img src={caseStudy.image} draggable={false} alt="" />
                     <div className={styles.cardImageBackground} style={{ backgroundImage: `url(${caseStudy.backgroundImage})` }} />
                   </div>
                   <div className={styles.card_heading_div}>
diff --git a/docs-website/src/pages/_components/Community/community.module.scss b/docs-website/src/pages/_components/Community/community.module.scss
index 62f0dc13b110a2..65da50435597db 100644
--- a/docs-website/src/pages/_components/Community/community.module.scss
+++ b/docs-website/src/pages/_components/Community/community.module.scss
@@ -221,14 +221,14 @@
 
 .numberContainer {
   display: inline-block;
-  width: 11rem;
+  width: 12rem;
   text-align: right;
 }
 
 .numberChange {
   display: inline-block;
   animation: slideIn 0.5s ease-in-out;
-  width: 11rem;
+  width: 12rem;
 }
 
 
diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss
index 1c1f13cbca566d..27bdca710b821d 100644
--- a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss
+++ b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss
@@ -129,7 +129,7 @@
 
     .quickstart__content {
       display: flex;
-      margin-bottom: 3rem;
+      margin-bottom: 6rem;
       width: 100%;
 
       .quickstart__text {
diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js
index 6e126adc4cb3a1..377fe0aa6475e3 100644
--- a/docs-website/src/pages/cloud/index.js
+++ b/docs-website/src/pages/cloud/index.js
@@ -1,4 +1,4 @@
-import React, { useState } from "react";
+import React, { useEffect, useState } from "react";
 import Layout from "@theme/Layout";
 import Link from "@docusaurus/Link";
 import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
@@ -10,8 +10,11 @@ import UnifiedTabs from "./UnifiedTabs";
 import FeatureCards from "./FeatureCards";
 import Hero from "./Hero";
 import DemoForm from "./DemoForm";
+import ServiceBell from "@servicebell/widget";
 import DemoFormModal from "./DemoFormModal";
 
+const SERVICE_BELL_ID = "00892146e5bc46d98d55ecc2b2fa67e2";
+
 function Home() {
   const context = useDocusaurusContext();
   const { siteConfig = {} } = context;
@@ -24,6 +27,10 @@ function Home() {
     window.location.replace("/docs");
   }
 
+  useEffect(() => {
+    ServiceBell("init", SERVICE_BELL_ID, { hidden: false });
+  }, []);
+
   return !siteConfig.customFields.isSaas ? (
     <Layout
       title={'DataHub Cloud - Unify Data Observability, Governance and Discovery'}
diff --git a/docs-website/src/pages/solutions/_components/Integrations/index.js b/docs-website/src/pages/solutions/_components/Integrations/index.js
index 77f028eb4cf74a..763c2a185c7ddb 100644
--- a/docs-website/src/pages/solutions/_components/Integrations/index.js
+++ b/docs-website/src/pages/solutions/_components/Integrations/index.js
@@ -47,6 +47,7 @@ const Integrations = () => {
               </div>
           </div>
       </div>
+      <a href="/integrations">See all →</a>
     </div>
   );
 };
diff --git a/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss b/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss
index da0c6964e8775f..07a6a2548e41a6 100644
--- a/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss
+++ b/docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss
@@ -1,6 +1,13 @@
 .container {
   display: flex;
   flex-direction: column;
+  >a {
+    text-decoration: none;
+    text-align: center;
+    margin-top: 1rem;
+    margin-bottom: 1rem;
+    font-size: 1.25rem;
+  }
 
   .section_header {
     color: var(--primitives-text-tex-subtext, #777E99);
diff --git a/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js b/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js
index 76b99b156704e0..17857f7b0360bd 100644
--- a/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js
+++ b/docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js
@@ -17,7 +17,7 @@ const Integrations = () => {
                 <div className={styles.slide_track}>
                   {[...Array(1)].map((_, i) => (
                     <React.Fragment key={i}>
-                      {[1, 2, 3, 4, 5, 6].map((item, index) => (
+                      {[1, 2, 3, 4, 5, 6, 7, 8].map((item, index) => (
                         <div className={styles.slide} key={index} style={{ backgroundImage: `url(${useBaseUrl(`${integrationsPath}/logo-integration-${item}.png`)})` }}>
                         </div>
                       ))}
diff --git a/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss b/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss
index aa2201fd0185c4..a945f4ae0598f4 100644
--- a/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss
+++ b/docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss
@@ -71,18 +71,22 @@
 
 .slider {
   position: relative;
+  display: flex;
 }
 
 .slide_track {
   display: flex;
-  width: max-content;
+  width: 80%;
   margin: auto;
+  flex-direction: row;
+  align-items: center;
+  justify-content: space-evenly;
 }
 
 .slide {
-  width: 100px;
-  height: 100px;
-  margin: auto 3rem;
+  width: 80px;
+  height: 80px;
+  margin: auto 0;
   display: flex;
   justify-content: space-between;
   overflow: hidden;
@@ -99,9 +103,12 @@
     max-width: 100vw;
     min-width: auto;
   }
+  .slide_track {
+    width: 95%;
+  }
   .slide {
-    width: 80px;
-    height: 80px;
-    margin: auto 1rem;
+    width: 40px;
+    height: 40px;
+    margin: auto 0;
   }
 }
\ No newline at end of file
diff --git a/docs-website/src/theme/Footer/index.js b/docs-website/src/theme/Footer/index.js
index 74e6eb7d09a7bc..fe0086993ee9eb 100644
--- a/docs-website/src/theme/Footer/index.js
+++ b/docs-website/src/theme/Footer/index.js
@@ -1,11 +1,15 @@
 import React from "react";
+import { useLocation } from "react-router-dom"; // Import useLocation from react-router-dom
 import Footer from "@theme-original/Footer";
 import MarkpromptHelp from "../../components/MarkpromptHelp";
 
 export default function FooterWrapper(props) {
+  const location = useLocation(); // Get the current location
+  const isDocsPage = location.pathname.startsWith("/docs"); // Check if the path starts with /docs
+
   return (
     <>
-      <MarkpromptHelp />
+      {isDocsPage && <MarkpromptHelp />}
       <Footer {...props} />
     </>
   );
diff --git a/docs-website/static/img/adoption-stories/adoption-stories-apple.png b/docs-website/static/img/adoption-stories/adoption-stories-apple.png
new file mode 100644
index 00000000000000..7ca5b06300abc5
Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-apple.png differ
diff --git a/docs-website/static/img/logos/companies/apple_text.png b/docs-website/static/img/logos/companies/apple_text.png
new file mode 100644
index 00000000000000..59ede054a96db5
Binary files /dev/null and b/docs-website/static/img/logos/companies/apple_text.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png
index acc17cb75d5856..c4d69b1ed3bc19 100644
Binary files a/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png and b/docs-website/static/img/solutions/integrations-observe/logo-integration-5.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png
index d9bb08766f5273..c22402e99d1f36 100644
Binary files a/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png and b/docs-website/static/img/solutions/integrations-observe/logo-integration-6.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-7.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-7.png
new file mode 100644
index 00000000000000..acc17cb75d5856
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-7.png differ
diff --git a/docs-website/static/img/solutions/integrations-observe/logo-integration-8.png b/docs-website/static/img/solutions/integrations-observe/logo-integration-8.png
new file mode 100644
index 00000000000000..d9bb08766f5273
Binary files /dev/null and b/docs-website/static/img/solutions/integrations-observe/logo-integration-8.png differ
diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock
index 0c4f2358d71398..4e457abd50af7a 100644
--- a/docs-website/yarn.lock
+++ b/docs-website/yarn.lock
@@ -1827,7 +1827,7 @@
     "@docusaurus/theme-search-algolia" "2.4.3"
     "@docusaurus/types" "2.4.3"
 
-"@docusaurus/react-loadable@5.5.2":
+"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2":
   version "5.5.2"
   resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce"
   integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
@@ -2843,6 +2843,11 @@
     rc-resize-observer "^1.3.1"
     rc-util "^5.38.0"
 
+"@servicebell/widget@^0.1.6":
+  version "0.1.6"
+  resolved "https://registry.yarnpkg.com/@servicebell/widget/-/widget-0.1.6.tgz#04672a7e7b14ff7025ec83fd740373345c359d74"
+  integrity sha512-Kh21FAETJlk32MRXHGYKxFTWLbbLlgLX3lKYH/02KHaQGIPRdHh/Ok7DiRc+/6f/OTQq3x3ady7e4o9weV5yQg==
+
 "@sideway/address@^4.1.5":
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.5.tgz#4bc149a0076623ced99ca8208ba780d65a99b9d5"
@@ -9708,14 +9713,6 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1:
   dependencies:
     "@babel/runtime" "^7.10.3"
 
-"react-loadable@npm:@docusaurus/react-loadable@5.5.2":
-  version "5.5.2"
-  resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce"
-  integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==
-  dependencies:
-    "@types/react" "*"
-    prop-types "^15.6.2"
-
 react-markdown@^8.0.6:
   version "8.0.7"
   resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b"
@@ -9830,6 +9827,11 @@ react-textarea-autosize@^8.3.2:
     use-composed-ref "^1.3.0"
     use-latest "^1.2.1"
 
+react-use-draggable-scroll@^0.4.7:
+  version "0.4.7"
+  resolved "https://registry.yarnpkg.com/react-use-draggable-scroll/-/react-use-draggable-scroll-0.4.7.tgz#86e77caab921ca07b134e9e1d1bc1810aeee4916"
+  integrity sha512-6gCxGPO9WV5dIsBaDrgUKBaac8CY07PkygcArfajijYSNDwAq0girDRjaBuF1+lRqQryoLFQfpVaV2u/Yh6CrQ==
+
 react-waypoint@^10.3.0:
   version "10.3.0"
   resolved "https://registry.yarnpkg.com/react-waypoint/-/react-waypoint-10.3.0.tgz#fcc60e86c6c9ad2174fa58d066dc6ae54e3df71d"
diff --git a/docs/quick-ingestion-guides/bigquery/setup.md b/docs/quick-ingestion-guides/bigquery/setup.md
index 96850f2deb68ed..7144ad6403ce9e 100644
--- a/docs/quick-ingestion-guides/bigquery/setup.md
+++ b/docs/quick-ingestion-guides/bigquery/setup.md
@@ -30,7 +30,7 @@ Please refer to the BigQuery [Permissions](https://cloud.google.com/iam/docs/per
    * [BigQuery Job User](https://cloud.google.com/bigquery/docs/access-control#bigquery.jobUser)
    * [BigQuery Metadata Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer)
    * [BigQuery Resource Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) -> This role is for Table-Level Lineage and Usage extraction
-   * [Logs View Accessor](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Table-Level Lineage and Usage extraction
+   * [Logs View Accessor](https://cloud.google.com/logging/docs/access-control#logging.viewAccessor) -> This role is for Table-Level Lineage and Usage extraction
    * [BigQuery Data Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Profiling
    * [BigQuery Read Session User](https://cloud.google.com/bigquery/docs/access-control#bigquery.readSessionUser) -> This role is for Profiling
 
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java
index 885be5e00b9454..41ef9c25a0f3eb 100644
--- a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java
+++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java
@@ -6,7 +6,6 @@
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX;
 import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD;
-import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX;
 
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.Status;
@@ -120,12 +119,13 @@ public static Optional<String> toStructuredPropertyFacetName(
       lookupDefinitionFromFilterOrFacetName(
           @Nonnull String fieldOrFacetName, @Nullable AspectRetriever aspectRetriever) {
     if (fieldOrFacetName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) {
-      String fqn =
-          fieldOrFacetName
-              .substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)
-              .replace(".keyword", "")
-              .replace(".delimited", "");
+      // Coming in from the UI this is structuredProperties.<FQN> + any particular specifier for
+      // subfield (.keyword etc)
+      String fqn = fieldOrFacetToFQN(fieldOrFacetName);
+
+      // FQN Maps directly to URN with urn:li:structuredProperties:FQN
       Urn urn = toURNFromFQN(fqn);
+
       Map<Urn, Map<String, Aspect>> result =
           Objects.requireNonNull(aspectRetriever)
               .getLatestAspectObjects(
@@ -223,9 +223,8 @@ public static void validateStructuredPropertyFQN(
    * @param fqn structured property's fqn
    * @return the expected structured property urn
    */
-  private static Urn toURNFromFQN(@Nonnull String fqn) {
-    return UrnUtils.getUrn(
-        String.join(":", "urn:li", STRUCTURED_PROPERTY_ENTITY_NAME, fqn.replace('_', '.')));
+  public static Urn toURNFromFQN(@Nonnull String fqn) {
+    return UrnUtils.getUrn(String.join(":", "urn:li", STRUCTURED_PROPERTY_ENTITY_NAME, fqn));
   }
 
   public static void validateFilter(
@@ -235,12 +234,13 @@ public static void validateFilter(
       return;
     }
 
-    Set<String> fieldNames = new HashSet<>();
+    Set<String> fqns = new HashSet<>();
 
     if (filter.getCriteria() != null) {
       for (Criterion c : filter.getCriteria()) {
         if (c.getField().startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
-          fieldNames.add(stripStructuredPropertyPrefix(c.getField()));
+          String fqn = fieldOrFacetToFQN(c.getField());
+          fqns.add(fqn);
         }
       }
     }
@@ -249,24 +249,23 @@ public static void validateFilter(
       for (ConjunctiveCriterion cc : filter.getOr()) {
         for (Criterion c : cc.getAnd()) {
           if (c.getField().startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
-            fieldNames.add(stripStructuredPropertyPrefix(c.getField()));
+            String fqn = fieldOrFacetToFQN(c.getField());
+            fqns.add(fqn);
           }
         }
       }
     }
 
-    if (!fieldNames.isEmpty()) {
-      validateStructuredPropertyFQN(fieldNames, Objects.requireNonNull(aspectRetriever));
+    if (!fqns.isEmpty()) {
+      validateStructuredPropertyFQN(fqns, Objects.requireNonNull(aspectRetriever));
     }
   }
 
-  private static String stripStructuredPropertyPrefix(String s) {
-    if (s.startsWith(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX)) {
-      return s.substring(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD.length() + 1).split("[.]")[0];
-    } else if (s.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) {
-      return s.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1).split("[.]")[0];
-    }
-    return s;
+  private static String fieldOrFacetToFQN(String fieldOrFacet) {
+    return fieldOrFacet
+        .substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)
+        .replace(".keyword", "")
+        .replace(".delimited", "");
   }
 
   public static Date toDate(PrimitivePropertyValue value) throws DateTimeParseException {
diff --git a/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml b/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml
index 21734cd4e03fa4..05e5205f7da419 100644
--- a/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml
+++ b/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml
@@ -22,3 +22,14 @@ source:
     soft_deleted_entities_cleanup:
       # Delete soft deleted entities which were deleted 10 days ago
       retention_days: 10
+    execution_request_cleanup:
+      # Minimum number of execution requests to keep, per ingestion source
+      keep_history_min_count: 10
+      # Maximum number of execution requests to keep, per ingestion source
+      keep_history_max_count: 1000
+      # Maximum number of days to keep execution requests for, per ingestion source
+      keep_history_max_days: 30
+      # Number of records per read operation
+      batch_read_size: 100
+      # Global switch for this cleanup task 
+      enabled: true
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index f81a0bbab03fe3..a16281aa07a32c 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -173,7 +173,7 @@
 
 looker_common = {
     # Looker Python SDK
-    "looker-sdk==23.0.0",
+    "looker-sdk>=23.0.0",
     # This version of lkml contains a fix for parsing lists in
     # LookML files with spaces between an item and the following comma.
     # See https://github.com/joshtemple/lkml/issues/73.
@@ -276,6 +276,13 @@
     *path_spec_common,
 }
 
+threading_timeout_common = {
+    "stopit==1.1.2",
+    # stopit uses pkg_resources internally, which means there's an implied
+    # dependency on setuptools.
+    "setuptools",
+}
+
 abs_base = {
     "azure-core==1.29.4",
     "azure-identity>=1.17.1",
@@ -484,11 +491,22 @@
     "teradata": sql_common
     | usage_common
     | sqlglot_lib
-    | {"teradatasqlalchemy>=17.20.0.0"},
+    | {
+        # On 2024-10-30, teradatasqlalchemy 20.0.0.2 was released. This version seemed to cause issues
+        # in our CI, so we're pinning the version for now.
+        "teradatasqlalchemy>=17.20.0.0,<=20.0.0.2",
+    },
     "trino": sql_common | trino,
     "starburst-trino-usage": sql_common | usage_common | trino,
     "nifi": {"requests", "packaging", "requests-gssapi"},
-    "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"} | sqlglot_lib,
+    "powerbi": (
+        (
+            microsoft_common
+            | {"lark[regex]==1.1.4", "sqlparse", "more-itertools"}
+            | sqlglot_lib
+            | threading_timeout_common
+        )
+    ),
     "powerbi-report-server": powerbi_report_server,
     "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.2"},
     "unity-catalog": databricks | sql_common | sqllineage_lib,
diff --git a/metadata-ingestion/src/datahub/api/entities/forms/forms.py b/metadata-ingestion/src/datahub/api/entities/forms/forms.py
index 9188ea33d6c684..fa6453851e0e74 100644
--- a/metadata-ingestion/src/datahub/api/entities/forms/forms.py
+++ b/metadata-ingestion/src/datahub/api/entities/forms/forms.py
@@ -16,8 +16,12 @@
 )
 from datahub.configuration.common import ConfigModel
 from datahub.emitter.mce_builder import (
+    make_container_urn,
     make_data_platform_urn,
+    make_domain_urn,
     make_group_urn,
+    make_tag_urn,
+    make_term_urn,
     make_user_urn,
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -36,6 +40,16 @@
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+FILTER_CRITERION_TYPES = "_entityType"
+FILTER_CRITERION_SUB_TYPES = "typeNames.keyword"
+FILTER_CRITERION_PLATFORMS = "platform.keyword"
+FILTER_CRITERION_PLATFORM_INSTANCES = "dataPlatformInstance.keyword"
+FILTER_CRITERION_DOMAINS = "domains.keyword"
+FILTER_CRITERION_CONTAINERS = "container.keyword"
+FILTER_CRITERION_OWNERS = "owners.keyword"
+FILTER_CRITERION_TAGS = "tags.keyword"
+FILTER_CRITERION_GLOSSARY_TERMS = "glossaryTerms.keyword"
+
 
 class PromptType(Enum):
     STRUCTURED_PROPERTY = "STRUCTURED_PROPERTY"
@@ -73,9 +87,14 @@ def has_value(cls, value):
 
 class Filters(ConfigModel):
     types: Optional[List[str]] = None
+    sub_types: Optional[List[str]] = None
     platforms: Optional[List[str]] = None
+    platform_instances: Optional[List[str]] = None
     domains: Optional[List[str]] = None
     containers: Optional[List[str]] = None
+    owners: Optional[List[str]] = None
+    tags: Optional[List[str]] = None
+    terms: Optional[List[str]] = None
 
 
 class Entities(ConfigModel):
@@ -105,7 +124,8 @@ class Forms(ConfigModel):
     @validator("urn", pre=True, always=True)
     def urn_must_be_present(cls, v, values):
         if not v:
-            assert values.get("id") is not None, "Form id must be present if urn is not"
+            if values.get("id") is None:
+                raise ValueError("Form id must be present if urn is not")
             return f"urn:li:form:{values['id']}"
         return v
 
@@ -249,29 +269,70 @@ def create_form_filters(self, emitter: DataHubGraph) -> Union[None, Exception]:
         # Loop through each entity and assign a filter for it
         if self.entities and self.entities.filters:
             filters = self.entities.filters
+
             if filters.types:
                 filters_raw.append(
-                    Forms.format_form_filter("_entityType", filters.types)
+                    Forms.format_form_filter(FILTER_CRITERION_TYPES, filters.types)
+                )
+
+            if filters.sub_types:
+                filters_raw.append(
+                    Forms.format_form_filter(
+                        FILTER_CRITERION_SUB_TYPES, filters.sub_types
+                    )
                 )
+
             if filters.platforms:
                 urns = [
                     make_data_platform_urn(platform) for platform in filters.platforms
                 ]
-                filters_raw.append(Forms.format_form_filter("platform", urns))
-            if filters.domains:
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_PLATFORMS, urns)
+                )
+
+            if filters.platform_instances:
                 urns = []
-                for domain in filters.domains:
-                    domain_urn = Forms.validate_domain_urn(domain)
-                    if domain_urn:
-                        urns.append(domain_urn)
-                filters_raw.append(Forms.format_form_filter("domains", urns))
+                for platform_instance in filters.platform_instances:
+                    platform_instance_urn = Forms.validate_platform_instance_urn(
+                        platform_instance
+                    )
+                    if platform_instance_urn:
+                        urns.append(platform_instance_urn)
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_PLATFORM_INSTANCES, urns)
+                )
+
+            if filters.domains:
+                urns = [make_domain_urn(domain) for domain in filters.domains]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_DOMAINS, urns)
+                )
+
             if filters.containers:
-                urns = []
-                for container in filters.containers:
-                    container_urn = Forms.validate_container_urn(container)
-                    if container_urn:
-                        urns.append(container_urn)
-                filters_raw.append(Forms.format_form_filter("container", urns))
+                urns = [
+                    make_container_urn(container) for container in filters.containers
+                ]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_CONTAINERS, urns)
+                )
+
+            if filters.owners:
+                urns = [make_user_urn(owner) for owner in filters.owners]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_OWNERS, urns)
+                )
+
+            if filters.tags:
+                urns = [make_tag_urn(tag) for tag in filters.tags]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_TAGS, urns)
+                )
+
+            if filters.terms:
+                urns = [make_term_urn(term) for term in filters.terms]
+                filters_raw.append(
+                    Forms.format_form_filter(FILTER_CRITERION_GLOSSARY_TERMS, urns)
+                )
 
         filters_str = ", ".join(item for item in filters_raw)
         result = emitter.execute_graphql(
@@ -279,6 +340,7 @@ def create_form_filters(self, emitter: DataHubGraph) -> Union[None, Exception]:
                 form_urn=self.urn, filters=filters_str
             )
         )
+
         if not result:
             return Exception(
                 f"Could not bulk upload urns or filters for form {self.urn}."
@@ -314,25 +376,20 @@ def format_form_filter(field: str, urns: List[str]) -> str:
         return FIELD_FILTER_TEMPLATE.format(field=field, values=formatted_urns)
 
     @staticmethod
-    def validate_domain_urn(domain: str) -> Union[str, None]:
-        if domain.startswith("urn:li:domain:"):
-            return domain
+    def validate_platform_instance_urn(instance: str) -> Union[str, None]:
+        if instance.startswith("urn:li:dataPlatformInstance:"):
+            return instance
 
-        logger.warning(f"{domain} is not an urn. Unable to create domain filter.")
-        return None
-
-    @staticmethod
-    def validate_container_urn(container: str) -> Union[str, None]:
-        if container.startswith("urn:li:container:"):
-            return container
-
-        logger.warning(f"{container} is not an urn. Unable to create container filter.")
+        logger.warning(
+            f"{instance} is not an urn. Unable to create platform instance filter."
+        )
         return None
 
     @staticmethod
     def from_datahub(graph: DataHubGraph, urn: str) -> "Forms":
         form: Optional[FormInfoClass] = graph.get_aspect(urn, FormInfoClass)
-        assert form is not None
+        if form is None:
+            raise Exception("FormInfo aspect is None. Unable to create form.")
         prompts = []
         for prompt_raw in form.prompts:
             prompts.append(
diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
index dab486c7f6f175..44fd33beadec9c 100644
--- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
+++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
@@ -89,7 +89,8 @@ def fqn(self) -> str:
     @validator("urn", pre=True, always=True)
     def urn_must_be_present(cls, v, values):
         if not v:
-            assert "id" in values, "id must be present if urn is not"
+            if "id" not in values:
+                raise ValueError("id must be present if urn is not")
             return f"urn:li:structuredProperty:{values['id']}"
         return v
 
@@ -153,7 +154,10 @@ def from_datahub(cls, graph: DataHubGraph, urn: str) -> "StructuredProperties":
         structured_property: Optional[
             StructuredPropertyDefinitionClass
         ] = graph.get_aspect(urn, StructuredPropertyDefinitionClass)
-        assert structured_property is not None
+        if structured_property is None:
+            raise Exception(
+                "StructuredPropertyDefinition aspect is None. Unable to create structured property."
+            )
         return StructuredProperties(
             urn=urn,
             qualified_name=structured_property.qualifiedName,
diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py
index 584bfc1f7cda56..8b852513e03c0f 100644
--- a/metadata-ingestion/src/datahub/cli/delete_cli.py
+++ b/metadata-ingestion/src/datahub/cli/delete_cli.py
@@ -213,6 +213,17 @@ def references(urn: str, dry_run: bool, force: bool) -> None:
         logger.info(f"Deleted {references_count} references to {urn}")
 
 
+@delete.command()
+@click.option("--urn", required=True, type=str, help="the urn of the entity")
+def undo_by_filter(urn: str) -> None:
+    """
+    Undo a soft deletion of an entity
+    """
+    graph = get_default_graph()
+    logger.info(f"Using {graph}")
+    graph.set_soft_delete_status(urn=urn, delete=False)
+
+
 @delete.command(no_args_is_help=True)
 @click.option(
     "--urn",
diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py
index 8654e6c282ba83..51f095751f7dd9 100644
--- a/metadata-ingestion/src/datahub/cli/ingest_cli.py
+++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py
@@ -1,4 +1,3 @@
-import asyncio
 import csv
 import json
 import logging
@@ -24,6 +23,7 @@
 from datahub.ingestion.run.pipeline import Pipeline
 from datahub.telemetry import telemetry
 from datahub.upgrade import upgrade
+from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
 
@@ -126,7 +126,7 @@ def run(
 ) -> None:
     """Ingest metadata into DataHub."""
 
-    async def run_pipeline_to_completion(pipeline: Pipeline) -> int:
+    def run_pipeline_to_completion(pipeline: Pipeline) -> int:
         logger.info("Starting metadata ingestion")
         with click_spinner.spinner(disable=no_spinner or no_progress):
             try:
@@ -166,44 +166,25 @@ async def run_pipeline_to_completion(pipeline: Pipeline) -> int:
         # The default is "datahub" reporting. The extra flag will disable it.
         report_to = None
 
-    async def run_ingestion_and_check_upgrade() -> int:
-        # TRICKY: We want to make sure that the Pipeline.create() call happens on the
-        # same thread as the rest of the ingestion. As such, we must initialize the
-        # pipeline inside the async function so that it happens on the same event
-        # loop, and hence the same thread.
-
-        # logger.debug(f"Using config: {pipeline_config}")
-        pipeline = Pipeline.create(
-            pipeline_config,
-            dry_run=dry_run,
-            preview_mode=preview,
-            preview_workunits=preview_workunits,
-            report_to=report_to,
-            no_progress=no_progress,
-            raw_config=raw_pipeline_config,
-        )
+    # logger.debug(f"Using config: {pipeline_config}")
+    pipeline = Pipeline.create(
+        pipeline_config,
+        dry_run=dry_run,
+        preview_mode=preview,
+        preview_workunits=preview_workunits,
+        report_to=report_to,
+        no_progress=no_progress,
+        raw_config=raw_pipeline_config,
+    )
+    with PerfTimer() as timer:
+        ret = run_pipeline_to_completion(pipeline)
 
-        version_stats_future = asyncio.ensure_future(
-            upgrade.retrieve_version_stats(pipeline.ctx.graph)
+    # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
+    if ret == 0:
+        upgrade.check_upgrade_post(
+            main_method_runtime=timer.elapsed_seconds(), graph=pipeline.ctx.graph
         )
-        ingestion_future = asyncio.ensure_future(run_pipeline_to_completion(pipeline))
-        ret = await ingestion_future
-
-        # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
-        if ret == 0:
-            try:
-                # we check the other futures quickly on success
-                version_stats = await asyncio.wait_for(version_stats_future, 0.5)
-                upgrade.maybe_print_upgrade_message(version_stats=version_stats)
-            except Exception as e:
-                logger.debug(
-                    f"timed out with {e} waiting for version stats to be computed... skipping ahead."
-                )
-
-        return ret
 
-    loop = asyncio.get_event_loop()
-    ret = loop.run_until_complete(run_ingestion_and_check_upgrade())
     if ret:
         sys.exit(ret)
     # don't raise SystemExit if there's no error
diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py
index d3a930d988171d..63b03db7f5b604 100644
--- a/metadata-ingestion/src/datahub/emitter/mce_builder.py
+++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py
@@ -163,11 +163,15 @@ def dataset_key_to_urn(key: DatasetKeyClass) -> str:
 
 
 def make_container_urn(guid: Union[str, "DatahubKey"]) -> str:
-    from datahub.emitter.mcp_builder import DatahubKey
+    if isinstance(guid, str) and guid.startswith("urn:li:container"):
+        return guid
+    else:
+        from datahub.emitter.mcp_builder import DatahubKey
+
+        if isinstance(guid, DatahubKey):
+            guid = guid.guid()
 
-    if isinstance(guid, DatahubKey):
-        guid = guid.guid()
-    return f"urn:li:container:{guid}"
+        return f"urn:li:container:{guid}"
 
 
 def container_urn_to_key(guid: str) -> Optional[ContainerKeyClass]:
diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py
index 1d2528a24c4e57..c90ac93eee2cc2 100644
--- a/metadata-ingestion/src/datahub/ingestion/graph/client.py
+++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py
@@ -1241,14 +1241,29 @@ def soft_delete_entity(
         Args:
             urn: The urn of the entity to soft-delete.
         """
+        self.set_soft_delete_status(
+            urn=urn, run_id=run_id, deletion_timestamp=deletion_timestamp, delete=True
+        )
+
+    def set_soft_delete_status(
+        self,
+        urn: str,
+        delete: bool,
+        run_id: str = _GRAPH_DUMMY_RUN_ID,
+        deletion_timestamp: Optional[int] = None,
+    ) -> None:
+        """Change status of soft-delete an entity by urn.
 
+        Args:
+            urn: The urn of the entity to soft-delete.
+        """
         assert urn
 
         deletion_timestamp = deletion_timestamp or int(time.time() * 1000)
         self.emit(
             MetadataChangeProposalWrapper(
                 entityUrn=urn,
-                aspect=StatusClass(removed=True),
+                aspect=StatusClass(removed=delete),
                 systemMetadata=SystemMetadataClass(
                     runId=run_id, lastObserved=deletion_timestamp
                 ),
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
index 4f18c22c108a6a..58317b108bef4c 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
@@ -175,7 +175,7 @@ def __init__(
 
     def get_query_result(self, query: str) -> RowIterator:
         def _should_retry(exc: BaseException) -> bool:
-            logger.debug(f"Exception occured for job query. Reason: {exc}")
+            logger.debug(f"Exception occurred for job query. Reason: {exc}")
             # Jobs sometimes fail with transient errors.
             # This is not currently handled by the python-bigquery client.
             # https://github.com/googleapis/python-bigquery/issues/23
@@ -197,7 +197,7 @@ def _should_retry(exc: BaseException) -> bool:
     def get_projects(self, max_results_per_page: int = 100) -> List[BigqueryProject]:
         def _should_retry(exc: BaseException) -> bool:
             logger.debug(
-                f"Exception occured for project.list api. Reason: {exc}. Retrying api request..."
+                f"Exception occurred for project.list api. Reason: {exc}. Retrying api request..."
             )
             self.report.num_list_projects_retry_request += 1
             return True
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
index 9705d63912b8db..a3304334cb1ebc 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/config.py
@@ -1,5 +1,5 @@
 import os
-from typing import Optional
+from typing import Optional, Set
 
 from pydantic import Field, root_validator
 
@@ -35,6 +35,19 @@ class DataHubSourceConfig(StatefulIngestionConfigBase):
         ),
     )
 
+    include_soft_deleted_entities: bool = Field(
+        default=True,
+        description=(
+            "If enabled, include entities that have been soft deleted. "
+            "Otherwise, include all entities regardless of removal status. "
+        ),
+    )
+
+    exclude_aspects: Set[str] = Field(
+        default_factory=set,
+        description="Set of aspect names to exclude from ingestion",
+    )
+
     database_query_batch_size: int = Field(
         default=DEFAULT_DATABASE_BATCH_SIZE,
         description="Number of records to fetch from the database at a time",
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
index 6986aac0a77573..382a0d548e38db 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_api_reader.py
@@ -26,11 +26,17 @@ def __init__(
         self.report = report
         self.graph = graph
 
-    def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
+    def get_urns(self) -> Iterable[str]:
         urns = self.graph.get_urns_by_filter(
-            status=RemovedStatusFilter.ALL,
+            status=RemovedStatusFilter.ALL
+            if self.config.include_soft_deleted_entities
+            else RemovedStatusFilter.NOT_SOFT_DELETED,
             batch_size=self.config.database_query_batch_size,
         )
+        return urns
+
+    def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
+        urns = self.get_urns()
         tasks: List[futures.Future[Iterable[MetadataChangeProposalWrapper]]] = []
         with futures.ThreadPoolExecutor(
             max_workers=self.config.max_workers
@@ -43,6 +49,9 @@ def get_aspects(self) -> Iterable[MetadataChangeProposalWrapper]:
     def _get_aspects_for_urn(self, urn: str) -> Iterable[MetadataChangeProposalWrapper]:
         aspects: Dict[str, _Aspect] = self.graph.get_entity_semityped(urn)  # type: ignore
         for aspect in aspects.values():
+            if aspect.get_aspect_name().lower() in self.config.exclude_aspects:
+                continue
+
             yield MetadataChangeProposalWrapper(
                 entityUrn=urn,
                 aspect=aspect,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
index e4f1bb275487ea..faa281097de4cd 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_database_reader.py
@@ -1,11 +1,10 @@
+import contextlib
 import json
 import logging
 from datetime import datetime
-from typing import Any, Generic, Iterable, List, Optional, Tuple, TypeVar
+from typing import Any, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
 
 from sqlalchemy import create_engine
-from sqlalchemy.engine import Row
-from typing_extensions import Protocol
 
 from datahub.emitter.aspect import ASPECT_MAP
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -21,13 +20,7 @@
 # Should work for at least mysql, mariadb, postgres
 DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
 
-
-class VersionOrderable(Protocol):
-    createdon: Any  # Should restrict to only orderable types
-    version: int
-
-
-ROW = TypeVar("ROW", bound=VersionOrderable)
+ROW = TypeVar("ROW", bound=Dict[str, Any])
 
 
 class VersionOrderer(Generic[ROW]):
@@ -54,14 +47,14 @@ def _process_row(self, row: ROW) -> Iterable[ROW]:
             return
 
         yield from self._attempt_queue_flush(row)
-        if row.version == 0:
+        if row["version"] == 0:
             self._add_to_queue(row)
         else:
             yield row
 
     def _add_to_queue(self, row: ROW) -> None:
         if self.queue is None:
-            self.queue = (row.createdon, [row])
+            self.queue = (row["createdon"], [row])
         else:
             self.queue[1].append(row)
 
@@ -69,7 +62,7 @@ def _attempt_queue_flush(self, row: ROW) -> Iterable[ROW]:
         if self.queue is None:
             return
 
-        if row.createdon > self.queue[0]:
+        if row["createdon"] > self.queue[0]:
             yield from self._flush_queue()
 
     def _flush_queue(self) -> Iterable[ROW]:
@@ -92,6 +85,21 @@ def __init__(
             **connection_config.options,
         )
 
+    @property
+    def soft_deleted_urns_query(self) -> str:
+        return f"""
+            SELECT DISTINCT mav.urn
+            FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)} as mav
+            JOIN (
+                SELECT *,
+                JSON_EXTRACT(metadata, '$.removed') as removed
+                FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
+                WHERE aspect = "status" AND version = 0
+            ) as sd ON sd.urn = mav.urn
+            WHERE sd.removed = true
+            ORDER BY mav.urn
+        """
+
     @property
     def query(self) -> str:
         # May repeat rows for the same date
@@ -101,66 +109,117 @@ def query(self) -> str:
         # Relies on createdon order to reflect version order
         # Ordering of entries with the same createdon is handled by VersionOrderer
         return f"""
-            SELECT urn, aspect, metadata, systemmetadata, createdon, version
-            FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
-            WHERE createdon >= %(since_createdon)s
-            {"" if self.config.include_all_versions else "AND version = 0"}
-            ORDER BY createdon, urn, aspect, version
-            LIMIT %(limit)s
-            OFFSET %(offset)s
+        SELECT *
+        FROM (
+            SELECT
+                mav.urn,
+                mav.aspect,
+                mav.metadata,
+                mav.systemmetadata,
+                mav.createdon,
+                mav.version,
+                removed
+            FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)} as mav
+            LEFT JOIN (
+                SELECT
+                    *,
+                    JSON_EXTRACT(metadata, '$.removed') as removed
+                FROM {self.engine.dialect.identifier_preparer.quote(self.config.database_table_name)}
+                WHERE aspect = 'status'
+                AND version = 0
+            ) as sd ON sd.urn = mav.urn
+            WHERE 1 = 1
+                {"" if self.config.include_all_versions else "AND mav.version = 0"}
+                {"" if not self.config.exclude_aspects else "AND mav.aspect NOT IN %(exclude_aspects)s"}
+                AND mav.createdon >= %(since_createdon)s
+            ORDER BY
+                createdon,
+                urn,
+                aspect,
+                version
+        ) as t
+        WHERE 1=1
+            {"" if self.config.include_soft_deleted_entities else "AND (removed = false or removed is NULL)"}
+        ORDER BY
+            createdon,
+            urn,
+            aspect,
+            version
         """
 
     def get_aspects(
         self, from_createdon: datetime, stop_time: datetime
     ) -> Iterable[Tuple[MetadataChangeProposalWrapper, datetime]]:
-        orderer = VersionOrderer[Row](enabled=self.config.include_all_versions)
+        orderer = VersionOrderer[Dict[str, Any]](
+            enabled=self.config.include_all_versions
+        )
         rows = self._get_rows(from_createdon=from_createdon, stop_time=stop_time)
         for row in orderer(rows):
             mcp = self._parse_row(row)
             if mcp:
-                yield mcp, row.createdon
+                yield mcp, row["createdon"]
 
-    def _get_rows(self, from_createdon: datetime, stop_time: datetime) -> Iterable[Row]:
+    def _get_rows(
+        self, from_createdon: datetime, stop_time: datetime
+    ) -> Iterable[Dict[str, Any]]:
         with self.engine.connect() as conn:
-            ts = from_createdon
-            offset = 0
-            while ts.timestamp() <= stop_time.timestamp():
-                logger.debug(f"Polling database aspects from {ts}")
-                rows = conn.execute(
+            with contextlib.closing(conn.connection.cursor()) as cursor:
+                cursor.execute(
                     self.query,
-                    since_createdon=ts.strftime(DATETIME_FORMAT),
-                    limit=self.config.database_query_batch_size,
-                    offset=offset,
+                    {
+                        "exclude_aspects": list(self.config.exclude_aspects),
+                        "since_createdon": from_createdon.strftime(DATETIME_FORMAT),
+                    },
                 )
-                if not rows.rowcount:
-                    return
 
-                for i, row in enumerate(rows):
-                    yield row
+                columns = [desc[0] for desc in cursor.description]
+                while True:
+                    rows = cursor.fetchmany(self.config.database_query_batch_size)
+                    if not rows:
+                        return
+                    for row in rows:
+                        yield dict(zip(columns, row))
 
-                if ts == row.createdon:
-                    offset += i + 1
-                else:
-                    ts = row.createdon
-                    offset = 0
+    def get_soft_deleted_rows(self) -> Iterable[Dict[str, Any]]:
+        """
+        Fetches all soft-deleted entities from the database.
 
-    def _parse_row(self, row: Row) -> Optional[MetadataChangeProposalWrapper]:
+        Yields:
+            Row objects containing URNs of soft-deleted entities
+        """
+        with self.engine.connect() as conn:
+            with contextlib.closing(conn.connection.cursor()) as cursor:
+                logger.debug("Polling soft-deleted urns from database")
+                cursor.execute(self.soft_deleted_urns_query)
+                columns = [desc[0] for desc in cursor.description]
+                while True:
+                    rows = cursor.fetchmany(self.config.database_query_batch_size)
+                    if not rows:
+                        return
+                    for row in rows:
+                        yield dict(zip(columns, row))
+
+    def _parse_row(
+        self, row: Dict[str, Any]
+    ) -> Optional[MetadataChangeProposalWrapper]:
         try:
-            json_aspect = post_json_transform(json.loads(row.metadata))
-            json_metadata = post_json_transform(json.loads(row.systemmetadata or "{}"))
+            json_aspect = post_json_transform(json.loads(row["metadata"]))
+            json_metadata = post_json_transform(
+                json.loads(row["systemmetadata"] or "{}")
+            )
             system_metadata = SystemMetadataClass.from_obj(json_metadata)
             return MetadataChangeProposalWrapper(
-                entityUrn=row.urn,
-                aspect=ASPECT_MAP[row.aspect].from_obj(json_aspect),
+                entityUrn=row["urn"],
+                aspect=ASPECT_MAP[row["aspect"]].from_obj(json_aspect),
                 systemMetadata=system_metadata,
                 changeType=ChangeTypeClass.UPSERT,
             )
         except Exception as e:
             logger.warning(
-                f"Failed to parse metadata for {row.urn}: {e}", exc_info=True
+                f'Failed to parse metadata for {row["urn"]}: {e}', exc_info=True
             )
             self.report.num_database_parse_errors += 1
             self.report.database_parse_errors.setdefault(
                 str(e), LossyDict()
-            ).setdefault(row.aspect, LossyList()).append(row.urn)
+            ).setdefault(row["aspect"], LossyList()).append(row["urn"])
             return None
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py
index d9e53e87c2cea9..56a3d55abb184f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py
@@ -36,6 +36,7 @@ def __init__(
         self.connection_config = connection_config
         self.report = report
         self.group_id = f"{KAFKA_GROUP_PREFIX}-{ctx.pipeline_name}"
+        self.ctx = ctx
 
     def __enter__(self) -> "DataHubKafkaReader":
         self.consumer = DeserializingConsumer(
@@ -95,6 +96,10 @@ def _poll_partition(
                 )
                 break
 
+            if mcl.aspectName and mcl.aspectName in self.config.exclude_aspects:
+                self.report.num_kafka_excluded_aspects += 1
+                continue
+
             # TODO: Consider storing state in kafka instead, via consumer.commit()
             yield mcl, PartitionOffset(partition=msg.partition(), offset=msg.offset())
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
index 7d2d99dbb3b23a..63cea45f75864b 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
@@ -62,13 +62,18 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         self.report.stop_time = datetime.now(tz=timezone.utc)
         logger.info(f"Ingesting DataHub metadata up until {self.report.stop_time}")
         state = self.stateful_ingestion_handler.get_last_run_state()
+        database_reader: Optional[DataHubDatabaseReader] = None
 
         if self.config.pull_from_datahub_api:
             yield from self._get_api_workunits()
 
         if self.config.database_connection is not None:
+            database_reader = DataHubDatabaseReader(
+                self.config, self.config.database_connection, self.report
+            )
+
             yield from self._get_database_workunits(
-                from_createdon=state.database_createdon_datetime
+                from_createdon=state.database_createdon_datetime, reader=database_reader
             )
             self._commit_progress()
         else:
@@ -77,7 +82,19 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             )
 
         if self.config.kafka_connection is not None:
-            yield from self._get_kafka_workunits(from_offsets=state.kafka_offsets)
+            soft_deleted_urns = []
+            if not self.config.include_soft_deleted_entities:
+                if database_reader is None:
+                    raise ValueError(
+                        "Cannot exclude soft deleted entities without a database connection"
+                    )
+                soft_deleted_urns = [
+                    row["urn"] for row in database_reader.get_soft_deleted_rows()
+                ]
+
+            yield from self._get_kafka_workunits(
+                from_offsets=state.kafka_offsets, soft_deleted_urns=soft_deleted_urns
+            )
             self._commit_progress()
         else:
             logger.info(
@@ -85,15 +102,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             )
 
     def _get_database_workunits(
-        self, from_createdon: datetime
+        self, from_createdon: datetime, reader: DataHubDatabaseReader
     ) -> Iterable[MetadataWorkUnit]:
-        if self.config.database_connection is None:
-            return
-
         logger.info(f"Fetching database aspects starting from {from_createdon}")
-        reader = DataHubDatabaseReader(
-            self.config, self.config.database_connection, self.report
-        )
         mcps = reader.get_aspects(from_createdon, self.report.stop_time)
         for i, (mcp, createdon) in enumerate(mcps):
             if not self.urn_pattern.allowed(str(mcp.entityUrn)):
@@ -112,20 +123,29 @@ def _get_database_workunits(
             self._commit_progress(i)
 
     def _get_kafka_workunits(
-        self, from_offsets: Dict[int, int]
+        self, from_offsets: Dict[int, int], soft_deleted_urns: List[str] = []
     ) -> Iterable[MetadataWorkUnit]:
         if self.config.kafka_connection is None:
             return
 
         logger.info("Fetching timeseries aspects from kafka")
         with DataHubKafkaReader(
-            self.config, self.config.kafka_connection, self.report, self.ctx
+            self.config,
+            self.config.kafka_connection,
+            self.report,
+            self.ctx,
         ) as reader:
             mcls = reader.get_mcls(
                 from_offsets=from_offsets, stop_time=self.report.stop_time
             )
             for i, (mcl, offset) in enumerate(mcls):
                 mcp = MetadataChangeProposalWrapper.try_from_mcl(mcl)
+                if mcp.entityUrn in soft_deleted_urns:
+                    self.report.num_timeseries_soft_deleted_aspects_dropped += 1
+                    logger.debug(
+                        f"Dropping soft-deleted aspect of {mcp.aspectName} on {mcp.entityUrn}"
+                    )
+                    continue
                 if mcp.changeType == ChangeTypeClass.DELETE:
                     self.report.num_timeseries_deletions_dropped += 1
                     logger.debug(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py
index 73e5a798a15533..721fc879894423 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/report.py
@@ -20,6 +20,8 @@ class DataHubSourceReport(StatefulIngestionReport):
 
     num_kafka_aspects_ingested: int = 0
     num_kafka_parse_errors: int = 0
+    num_kafka_excluded_aspects: int = 0
     kafka_parse_errors: LossyDict[str, int] = field(default_factory=LossyDict)
 
     num_timeseries_deletions_dropped: int = 0
+    num_timeseries_soft_deleted_aspects_dropped: int = 0
diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
index 1897f3f288ec0b..c4b4186f45fc38 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
@@ -24,6 +24,11 @@
     DataProcessCleanupConfig,
     DataProcessCleanupReport,
 )
+from datahub.ingestion.source.gc.execution_request_cleanup import (
+    DatahubExecutionRequestCleanup,
+    DatahubExecutionRequestCleanupConfig,
+    DatahubExecutionRequestCleanupReport,
+)
 from datahub.ingestion.source.gc.soft_deleted_entity_cleanup import (
     SoftDeletedEntitiesCleanup,
     SoftDeletedEntitiesCleanupConfig,
@@ -70,9 +75,18 @@ class DataHubGcSourceConfig(ConfigModel):
         description="Configuration for soft deleted entities cleanup",
     )
 
+    execution_request_cleanup: Optional[DatahubExecutionRequestCleanupConfig] = Field(
+        default=None,
+        description="Configuration for execution request cleanup",
+    )
+
 
 @dataclass
-class DataHubGcSourceReport(DataProcessCleanupReport, SoftDeletedEntitiesReport):
+class DataHubGcSourceReport(
+    DataProcessCleanupReport,
+    SoftDeletedEntitiesReport,
+    DatahubExecutionRequestCleanupReport,
+):
     expired_tokens_revoked: int = 0
 
 
@@ -97,6 +111,7 @@ def __init__(self, ctx: PipelineContext, config: DataHubGcSourceConfig):
         self.graph = ctx.require_graph("The DataHubGc source")
         self.dataprocess_cleanup: Optional[DataProcessCleanup] = None
         self.soft_deleted_entities_cleanup: Optional[SoftDeletedEntitiesCleanup] = None
+        self.execution_request_cleanup: Optional[DatahubExecutionRequestCleanup] = None
 
         if self.config.dataprocess_cleanup:
             self.dataprocess_cleanup = DataProcessCleanup(
@@ -109,6 +124,12 @@ def __init__(self, ctx: PipelineContext, config: DataHubGcSourceConfig):
                 self.report,
                 self.config.dry_run,
             )
+        if self.config.execution_request_cleanup:
+            self.execution_request_cleanup = DatahubExecutionRequestCleanup(
+                config=self.config.execution_request_cleanup,
+                graph=self.graph,
+                report=self.report,
+            )
 
     @classmethod
     def create(cls, config_dict, ctx):
@@ -130,6 +151,8 @@ def get_workunits_internal(
             yield from self.dataprocess_cleanup.get_workunits_internal()
         if self.soft_deleted_entities_cleanup:
             self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities()
+        if self.execution_request_cleanup:
+            self.execution_request_cleanup.run()
         yield from []
 
     def truncate_indices(self) -> None:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py
new file mode 100644
index 00000000000000..570df4e99ab13d
--- /dev/null
+++ b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py
@@ -0,0 +1,240 @@
+import logging
+import time
+from typing import Any, Dict, Iterator, Optional
+
+from pydantic import BaseModel, Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.ingestion.api.source import SourceReport
+from datahub.ingestion.graph.client import DataHubGraph
+
+logger = logging.getLogger(__name__)
+
+DATAHUB_EXECUTION_REQUEST_ENTITY_NAME = "dataHubExecutionRequest"
+DATAHUB_EXECUTION_REQUEST_KEY_ASPECT_NAME = "dataHubExecutionRequestKey"
+DATAHUB_EXECUTION_REQUEST_INPUT_ASPECT_NAME = "dataHubExecutionRequestInput"
+DATAHUB_EXECUTION_REQUEST_RESULT_ASPECT_NAME = "dataHubExecutionRequestResult"
+
+
+class DatahubExecutionRequestCleanupConfig(ConfigModel):
+    keep_history_min_count: int = Field(
+        10,
+        description="Minimum number of execution requests to keep, per ingestion source",
+    )
+
+    keep_history_max_count: int = Field(
+        1000,
+        description="Maximum number of execution requests to keep, per ingestion source",
+    )
+
+    keep_history_max_days: int = Field(
+        30,
+        description="Maximum number of days to keep execution requests for, per ingestion source",
+    )
+
+    batch_read_size: int = Field(
+        100,
+        description="Number of records per read operation",
+    )
+
+    enabled: bool = Field(
+        True,
+        description="Global switch for this cleanup task",
+    )
+
+    def keep_history_max_milliseconds(self):
+        return self.keep_history_max_days * 24 * 3600 * 1000
+
+
+class DatahubExecutionRequestCleanupReport(SourceReport):
+    execution_request_cleanup_records_read: int = 0
+    execution_request_cleanup_records_preserved: int = 0
+    execution_request_cleanup_records_deleted: int = 0
+    execution_request_cleanup_read_errors: int = 0
+    execution_request_cleanup_delete_errors: int = 0
+
+
+class CleanupRecord(BaseModel):
+    urn: str
+    request_id: str
+    status: str
+    ingestion_source: str
+    requested_at: int
+
+
+class DatahubExecutionRequestCleanup:
+    def __init__(
+        self,
+        graph: DataHubGraph,
+        report: DatahubExecutionRequestCleanupReport,
+        config: Optional[DatahubExecutionRequestCleanupConfig] = None,
+    ) -> None:
+
+        self.graph = graph
+        self.report = report
+        self.instance_id = int(time.time())
+
+        if config is not None:
+            self.config = config
+        else:
+            self.config = DatahubExecutionRequestCleanupConfig()
+
+    def _to_cleanup_record(self, entry: Dict) -> CleanupRecord:
+        input_aspect = (
+            entry.get("aspects", {})
+            .get(DATAHUB_EXECUTION_REQUEST_INPUT_ASPECT_NAME, {})
+            .get("value", {})
+        )
+        result_aspect = (
+            entry.get("aspects", {})
+            .get(DATAHUB_EXECUTION_REQUEST_RESULT_ASPECT_NAME, {})
+            .get("value", {})
+        )
+        key_aspect = (
+            entry.get("aspects", {})
+            .get(DATAHUB_EXECUTION_REQUEST_KEY_ASPECT_NAME, {})
+            .get("value", {})
+        )
+        return CleanupRecord(
+            urn=entry.get("urn"),
+            request_id=key_aspect.get("id"),
+            requested_at=input_aspect.get("requestedAt", 0),
+            status=result_aspect.get("status", "PENDING"),
+            ingestion_source=input_aspect.get("source", {}).get("ingestionSource", ""),
+        )
+
+    def _scroll_execution_requests(
+        self, overrides: Dict[str, Any] = {}
+    ) -> Iterator[CleanupRecord]:
+        headers: Dict[str, Any] = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        params = {
+            "aspectNames": [
+                DATAHUB_EXECUTION_REQUEST_KEY_ASPECT_NAME,
+                DATAHUB_EXECUTION_REQUEST_INPUT_ASPECT_NAME,
+                DATAHUB_EXECUTION_REQUEST_RESULT_ASPECT_NAME,
+            ],
+            "count": str(self.config.batch_read_size),
+            "sort": "requestTimeMs",
+            "sortOrder": "DESCENDING",
+            "systemMetadata": "false",
+            "skipCache": "true",
+        }
+        params.update(overrides)
+
+        while True:
+            try:
+                url = f"{self.graph.config.server}/openapi/v2/entity/{DATAHUB_EXECUTION_REQUEST_ENTITY_NAME}"
+                response = self.graph._session.get(url, headers=headers, params=params)
+                response.raise_for_status()
+                document = response.json()
+
+                entries = document.get("results", [])
+                for entry in entries:
+                    yield self._to_cleanup_record(entry)
+
+                if "scrollId" not in document:
+                    break
+                params["scrollId"] = document["scrollId"]
+            except Exception as e:
+                logger.error(
+                    f"ergc({self.instance_id}): failed to fetch next batch of execution requests: {e}"
+                )
+                self.report.execution_request_cleanup_read_errors += 1
+
+    def _scroll_garbage_records(self):
+        state: Dict[str, Dict] = {}
+
+        now_ms = int(time.time()) * 1000
+        running_guard_timeout = now_ms - 30 * 24 * 3600 * 1000
+
+        for entry in self._scroll_execution_requests():
+            self.report.execution_request_cleanup_records_read += 1
+            key = entry.ingestion_source
+
+            # Always delete corrupted records
+            if not key:
+                logger.warning(
+                    f"ergc({self.instance_id}): will delete corrupted entry with missing source key: {entry}"
+                )
+                yield entry
+                continue
+
+            if key not in state:
+                state[key] = {}
+                state[key]["cutoffTimestamp"] = (
+                    entry.requested_at - self.config.keep_history_max_milliseconds()
+                )
+
+            state[key]["count"] = state[key].get("count", 0) + 1
+
+            # Do not delete if number of requests is below minimum
+            if state[key]["count"] < self.config.keep_history_min_count:
+                self.report.execution_request_cleanup_records_preserved += 1
+                continue
+
+            # Do not delete if number of requests do not exceed allowed maximum,
+            # or the cutoff date.
+            if (state[key]["count"] < self.config.keep_history_max_count) and (
+                entry.requested_at > state[key]["cutoffTimestamp"]
+            ):
+                self.report.execution_request_cleanup_records_preserved += 1
+                continue
+
+            # Do not delete if status is RUNNING or PENDING and created within last month. If the record is >month old and it did not
+            # transition to a final state within that timeframe, it likely has no value.
+            if entry.requested_at > running_guard_timeout and entry.status in [
+                "RUNNING",
+                "PENDING",
+            ]:
+                self.report.execution_request_cleanup_records_preserved += 1
+                continue
+
+            # Otherwise delete current record
+            logger.info(
+                (
+                    f"ergc({self.instance_id}): going to delete {entry.request_id} in source {key}; "
+                    f"source count: {state[key]['count']}; "
+                    f"source cutoff: {state[key]['cutoffTimestamp']}; "
+                    f"record timestamp: {entry.requested_at}."
+                )
+            )
+            self.report.execution_request_cleanup_records_deleted += 1
+            yield entry
+
+    def _delete_entry(self, entry: CleanupRecord) -> None:
+        try:
+            logger.info(
+                f"ergc({self.instance_id}): going to delete ExecutionRequest {entry.request_id}"
+            )
+            self.graph.delete_entity(entry.urn, True)
+        except Exception as e:
+            self.report.execution_request_cleanup_delete_errors += 1
+            logger.error(
+                f"ergc({self.instance_id}): failed to delete ExecutionRequest {entry.request_id}: {e}"
+            )
+
+    def run(self) -> None:
+        if not self.config.enabled:
+            logger.info(
+                f"ergc({self.instance_id}): ExecutionRequest cleaner is disabled."
+            )
+            return
+
+        logger.info(
+            (
+                f"ergc({self.instance_id}): Starting cleanup of ExecutionRequest records; "
+                f"max days: {self.config.keep_history_max_days}, "
+                f"min records: {self.config.keep_history_min_count}, "
+                f"max records: {self.config.keep_history_max_count}."
+            )
+        )
+
+        for entry in self._scroll_garbage_records():
+            self._delete_entry(entry)
+
+        logger.info(
+            f"ergc({self.instance_id}): Finished cleanup of ExecutionRequest records."
+        )
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
index 8a3f8ed6131a21..09ade4c905bea0 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
@@ -19,6 +19,8 @@
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
+from datahub.utilities.lossy_collections import LossyList
+from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
 
@@ -176,11 +178,27 @@ class SupportedDataPlatform(Enum):
 
 @dataclass
 class PowerBiDashboardSourceReport(StaleEntityRemovalSourceReport):
+    all_workspace_count: int = 0
+    filtered_workspace_names: LossyList[str] = dataclass_field(
+        default_factory=LossyList
+    )
+    filtered_workspace_types: LossyList[str] = dataclass_field(
+        default_factory=LossyList
+    )
+
     dashboards_scanned: int = 0
     charts_scanned: int = 0
     filtered_dashboards: List[str] = dataclass_field(default_factory=list)
     filtered_charts: List[str] = dataclass_field(default_factory=list)
-    number_of_workspaces: int = 0
+
+    m_query_parse_timer: PerfTimer = dataclass_field(default_factory=PerfTimer)
+    m_query_parse_attempts: int = 0
+    m_query_parse_successes: int = 0
+    m_query_parse_timeouts: int = 0
+    m_query_parse_validation_errors: int = 0
+    m_query_parse_unexpected_character_errors: int = 0
+    m_query_parse_unknown_errors: int = 0
+    m_query_resolver_errors: int = 0
 
     def report_dashboards_scanned(self, count: int = 1) -> None:
         self.dashboards_scanned += count
@@ -194,9 +212,6 @@ def report_dashboards_dropped(self, model: str) -> None:
     def report_charts_dropped(self, view: str) -> None:
         self.filtered_charts.append(view)
 
-    def report_number_of_workspaces(self, number_of_workspaces: int) -> None:
-        self.number_of_workspaces = number_of_workspaces
-
 
 def default_for_dataset_type_mapping() -> Dict[str, str]:
     dict_: dict = {}
@@ -331,7 +346,7 @@ class PowerBiDashboardSourceConfig(
     )
     workspace_id_as_urn_part: bool = pydantic.Field(
         default=False,
-        description="It is recommended to set this to True only if you have legacy workspaces based on Office 365 groups, as those workspaces can have identical names."
+        description="It is recommended to set this to True only if you have legacy workspaces based on Office 365 groups, as those workspaces can have identical names. "
         "To maintain backward compatibility, this is set to False which uses workspace name",
     )
     # Enable/Disable extracting ownership information of Dashboard
@@ -371,8 +386,8 @@ class PowerBiDashboardSourceConfig(
     # any existing tags defined to those entities
     extract_endorsements_to_tags: bool = pydantic.Field(
         default=False,
-        description="Whether to extract endorsements to tags, note that this may overwrite existing tags. Admin API "
-        "access is required is this setting is enabled",
+        description="Whether to extract endorsements to tags, note that this may overwrite existing tags. "
+        "Admin API access is required if this setting is enabled.",
     )
     filter_dataset_endorsements: AllowDenyPattern = pydantic.Field(
         default=AllowDenyPattern.allow_all(),
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py
index a82384c6526b23..6b16d3d4f3c1fb 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py
@@ -1,6 +1,7 @@
 import functools
 import importlib.resources as pkg_resource
 import logging
+import os
 from typing import Dict, List
 
 import lark
@@ -19,9 +20,12 @@
     TRACE_POWERBI_MQUERY_PARSER,
 )
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table
+from datahub.utilities.threading_timeout import TimeoutException, threading_timeout
 
 logger = logging.getLogger(__name__)
 
+_M_QUERY_PARSE_TIMEOUT = int(os.getenv("DATAHUB_POWERBI_M_QUERY_PARSE_TIMEOUT", 60))
+
 
 @functools.lru_cache(maxsize=1)
 def get_lark_parser() -> Lark:
@@ -41,7 +45,8 @@ def _parse_expression(expression: str) -> Tree:
     expression = expression.replace("\u00a0", " ")
 
     logger.debug(f"Parsing expression = {expression}")
-    parse_tree: Tree = lark_parser.parse(expression)
+    with threading_timeout(_M_QUERY_PARSE_TIMEOUT):
+        parse_tree: Tree = lark_parser.parse(expression)
 
     if TRACE_POWERBI_MQUERY_PARSER:
         logger.debug(parse_tree.pretty())
@@ -68,7 +73,9 @@ def get_upstream_tables(
     )
 
     try:
-        parse_tree: Tree = _parse_expression(table.expression)
+        with reporter.m_query_parse_timer:
+            reporter.m_query_parse_attempts += 1
+            parse_tree: Tree = _parse_expression(table.expression)
 
         valid, message = validator.validate_parse_tree(
             parse_tree, native_query_enabled=config.native_query_parsing
@@ -81,26 +88,39 @@ def get_upstream_tables(
                 message="DataAccess function is not present in M-Query expression",
                 context=f"table-full-name={table.full_name}, expression={table.expression}, message={message}",
             )
+            reporter.m_query_parse_validation_errors += 1
             return []
+    except KeyboardInterrupt:
+        raise
+    except TimeoutException:
+        reporter.m_query_parse_timeouts += 1
+        reporter.warning(
+            title="M-Query Parsing Timeout",
+            message=f"M-Query parsing timed out after {_M_QUERY_PARSE_TIMEOUT} seconds. Lineage for this table will not be extracted.",
+            context=f"table-full-name={table.full_name}, expression={table.expression}",
+        )
+        return []
     except (
         BaseException
     ) as e:  # TODO: Debug why BaseException is needed here and below.
         if isinstance(e, lark.exceptions.UnexpectedCharacters):
-            title = "Unexpected Character Found"
+            error_type = "Unexpected Character Error"
+            reporter.m_query_parse_unexpected_character_errors += 1
         else:
-            title = "Unknown Parsing Error"
+            error_type = "Unknown Parsing Error"
+            reporter.m_query_parse_unknown_errors += 1
 
         reporter.warning(
-            title=title,
-            message="Unknown parsing error",
+            title="Unable to extract lineage from M-Query expression",
+            message=f"Got an '{error_type}' while parsing the expression. Lineage will be missing for this table.",
             context=f"table-full-name={table.full_name}, expression={table.expression}",
             exc=e,
         )
         return []
+    reporter.m_query_parse_successes += 1
 
-    lineage: List[resolver.Lineage] = []
     try:
-        lineage = resolver.MQueryResolver(
+        lineage: List[resolver.Lineage] = resolver.MQueryResolver(
             table=table,
             parse_tree=parse_tree,
             reporter=reporter,
@@ -111,14 +131,14 @@ def get_upstream_tables(
             platform_instance_resolver=platform_instance_resolver,
         )
 
+        return lineage
+
     except BaseException as e:
+        reporter.m_query_resolver_errors += 1
         reporter.warning(
             title="Unknown M-Query Pattern",
             message="Encountered a unknown M-Query Expression",
             context=f"table-full-name={table.full_name}, expression={table.expression}, message={e}",
             exc=e,
         )
-
-        logger.debug(f"Stack trace for {table.full_name}:", exc_info=e)
-
-    return lineage
+        return []
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py
index d8918da42ae338..2b33d3e4f9b2c3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py
@@ -353,8 +353,9 @@ def _process_invoke_expression(
             )
             if arg_list is None:
                 self.reporter.report_warning(
-                    f"{self.table.full_name}-arg-list",
-                    f"Argument list not found for data-access-function {data_access_func}",
+                    title="M-Query Resolver Error",
+                    message="Unable to extract lineage from parsed M-Query expression (missing argument list)",
+                    context=f"{self.table.full_name}: argument list not found for data-access-function {data_access_func}",
                 )
                 return None
 
@@ -374,8 +375,9 @@ def _process_invoke_expression(
                 f"Function invocation without argument in expression = {invoke_expression.pretty()}"
             )
             self.reporter.report_warning(
-                f"{self.table.full_name}-variable-statement",
-                "Function invocation without argument",
+                title="M-Query Resolver Error",
+                message="Unable to extract lineage from parsed M-Query expression (function invocation without argument)",
+                context=f"{self.table.full_name}: function invocation without argument",
             )
             return None
 
@@ -400,8 +402,9 @@ def _process_invoke_expression(
                     f"Either list_expression or type_expression is not found = {invoke_expression.pretty()}"
                 )
                 self.reporter.report_warning(
-                    f"{self.table.full_name}-variable-statement",
-                    "Function argument expression is not supported",
+                    title="M-Query Resolver Error",
+                    message="Unable to extract lineage from parsed M-Query expression (function argument expression is not supported)",
+                    context=f"{self.table.full_name}: function argument expression is not supported",
                 )
                 return None
 
@@ -470,8 +473,9 @@ def internal(
             )
             if v_statement is None:
                 self.reporter.report_warning(
-                    f"{self.table.full_name}-variable-statement",
-                    f"output variable ({current_identifier}) statement not found in table expression",
+                    title="Unable to extract lineage from M-Query expression",
+                    message="Lineage will be incomplete.",
+                    context=f"table-full-name={self.table.full_name}: output-variable={current_identifier} not found in table expression",
                 )
                 return None
 
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
index 72336afbaacd05..b2afdc3e409319 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -7,6 +7,8 @@
 from datetime import datetime
 from typing import Iterable, List, Optional, Tuple, Union
 
+import more_itertools
+
 import datahub.emitter.mce_builder as builder
 import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -795,6 +797,11 @@ def generate_container_for_workspace(
             container_key=self.workspace_key,
             name=workspace.name,
             sub_types=[workspace.type],
+            extra_properties={
+                "workspace_id": workspace.id,
+                "workspace_name": workspace.name,
+                "workspace_type": workspace.type,
+            },
         )
         return container_work_units
 
@@ -1256,20 +1263,33 @@ def create(cls, config_dict, ctx):
 
     def get_allowed_workspaces(self) -> List[powerbi_data_classes.Workspace]:
         all_workspaces = self.powerbi_client.get_workspaces()
+        logger.info(f"Number of workspaces = {len(all_workspaces)}")
+        self.reporter.all_workspace_count = len(all_workspaces)
+        logger.debug(
+            f"All workspaces: {[workspace.format_name_for_logger() for workspace in all_workspaces]}"
+        )
 
-        allowed_wrk = [
-            workspace
-            for workspace in all_workspaces
-            if self.source_config.workspace_id_pattern.allowed(workspace.id)
-            and workspace.type in self.source_config.workspace_type_filter
-        ]
+        allowed_workspaces = []
+        for workspace in all_workspaces:
+            if not self.source_config.workspace_id_pattern.allowed(workspace.id):
+                self.reporter.filtered_workspace_names.append(
+                    f"{workspace.id} - {workspace.name}"
+                )
+                continue
+            elif workspace.type not in self.source_config.workspace_type_filter:
+                self.reporter.filtered_workspace_types.append(
+                    f"{workspace.id} - {workspace.name} (type = {workspace.type})"
+                )
+                continue
+            else:
+                allowed_workspaces.append(workspace)
 
-        logger.info(f"Number of workspaces = {len(all_workspaces)}")
-        self.reporter.report_number_of_workspaces(len(all_workspaces))
-        logger.info(f"Number of allowed workspaces = {len(allowed_wrk)}")
-        logger.debug(f"Workspaces = {all_workspaces}")
+        logger.info(f"Number of allowed workspaces = {len(allowed_workspaces)}")
+        logger.debug(
+            f"Allowed workspaces: {[workspace.format_name_for_logger() for workspace in allowed_workspaces]}"
+        )
 
-        return allowed_wrk
+        return allowed_workspaces
 
     def validate_dataset_type_mapping(self):
         powerbi_data_platforms: List[str] = [
@@ -1480,16 +1500,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
         # Fetch PowerBi workspace for given workspace identifier
 
         allowed_workspaces = self.get_allowed_workspaces()
-        workspaces_len = len(allowed_workspaces)
-
-        batch_size = (
-            self.source_config.scan_batch_size
-        )  # 100 is the maximum allowed for powerbi scan
-        num_batches = (workspaces_len + batch_size - 1) // batch_size
-        batches = [
-            allowed_workspaces[i * batch_size : (i + 1) * batch_size]
-            for i in range(num_batches)
-        ]
+
+        batches = more_itertools.chunked(
+            allowed_workspaces, self.source_config.scan_batch_size
+        )
         for batch_workspaces in batches:
             for workspace in self.powerbi_client.fill_workspaces(
                 batch_workspaces, self.reporter
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
index 9407ef7a51b58e..fc5cd76458a510 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
@@ -96,6 +96,9 @@ def get_workspace_key(
             instance=platform_instance,
         )
 
+    def format_name_for_logger(self) -> str:
+        return f"{self.name} ({self.id})"
+
 
 @dataclass
 class DataSource:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
index f8fff2391d10b0..7a47c40976bec6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
@@ -811,7 +811,7 @@ def _is_scan_result_ready(
             res.raise_for_status()
 
             if res.json()[Constant.STATUS].upper() == Constant.SUCCEEDED:
-                logger.info(f"Scan result is available for scan id({scan_id})")
+                logger.debug(f"Scan result is available for scan id({scan_id})")
                 return True
 
             if retry == max_retry:
@@ -898,8 +898,8 @@ def get_users(self, workspace_id: str, entity: str, entity_id: str) -> List[User
         return users
 
     def get_scan_result(self, scan_id: str) -> Optional[dict]:
-        logger.info("Fetching scan result")
-        logger.info(f"{Constant.SCAN_ID}={scan_id}")
+        logger.debug("Fetching scan result")
+        logger.debug(f"{Constant.SCAN_ID}={scan_id}")
         scan_result_get_endpoint = AdminAPIResolver.API_ENDPOINTS[
             Constant.SCAN_RESULT_GET
         ]
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
index bdd76fb29fff4f..b38302c6aa2253 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
@@ -303,7 +303,7 @@ def _get_scan_result(self, workspace_ids: List[str]) -> Any:
                 )
             return None
 
-        logger.info("Waiting for scan to complete")
+        logger.debug("Waiting for scan to complete")
         if (
             self.__admin_api_resolver.wait_for_scan_to_complete(
                 scan_id=scan_id, timeout=self.__config.scan_timeout
@@ -355,22 +355,32 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict:
         logger.debug("Processing scan result for datasets")
 
         for dataset_dict in datasets:
-            dataset_instance: PowerBIDataset = self._get_resolver().get_dataset(
-                workspace=workspace,
-                dataset_id=dataset_dict[Constant.ID],
-            )
+            dataset_id = dataset_dict[Constant.ID]
+            try:
+                dataset_instance = self._get_resolver().get_dataset(
+                    workspace=workspace,
+                    dataset_id=dataset_id,
+                )
+                if dataset_instance is None:
+                    continue
+            except Exception as e:
+                self.reporter.warning(
+                    title="Unable to fetch dataset details",
+                    message="Skipping this dataset due to the error. Metadata will be incomplete.",
+                    context=f"workspace={workspace.name}, dataset-id={dataset_id}",
+                    exc=e,
+                )
+                continue
 
             # fetch + set dataset parameters
             try:
                 dataset_parameters = self._get_resolver().get_dataset_parameters(
                     workspace_id=workspace.id,
-                    dataset_id=dataset_dict[Constant.ID],
+                    dataset_id=dataset_id,
                 )
                 dataset_instance.parameters = dataset_parameters
             except Exception as e:
-                logger.info(
-                    f"Unable to fetch dataset parameters for {dataset_dict[Constant.ID]}: {e}"
-                )
+                logger.info(f"Unable to fetch dataset parameters for {dataset_id}: {e}")
 
             if self.__config.extract_endorsements_to_tags:
                 dataset_instance.tags = self._parse_endorsement(
@@ -564,8 +574,7 @@ def _fill_metadata_from_scan_result(
                 )
             else:
                 logger.info(
-                    "Skipping endorsements tag as extract_endorsements_to_tags is set to "
-                    "false "
+                    "Skipping endorsements tag as extract_endorsements_to_tags is not enabled"
                 )
 
             self._populate_app_details(
@@ -640,6 +649,9 @@ def fill_dashboard_tags() -> None:
     def fill_workspaces(
         self, workspaces: List[Workspace], reporter: PowerBiDashboardSourceReport
     ) -> Iterable[Workspace]:
+        logger.info(
+            f"Fetching initial metadata for workspaces: {[workspace.format_name_for_logger() for workspace in workspaces]}"
+        )
 
         workspaces = self._fill_metadata_from_scan_result(workspaces=workspaces)
         # First try to fill the admin detail as some regular metadata contains lineage to admin metadata
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
index e7cc5fc50cf6e5..4b84c25965a994 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
@@ -57,6 +57,7 @@ def __init__(
         self.context = context
 
         self.database = database
+        self.known_urns: Set[str] = set()  # will be set later
 
         self.aggregator = SqlParsingAggregator(
             platform=self.platform,
@@ -68,6 +69,7 @@ def __init__(
             generate_operations=False,
             usage_config=self.config,
             graph=self.context.graph,
+            is_temp_table=self._is_temp_table,
         )
         self.report.sql_aggregator = self.aggregator.report
 
@@ -87,7 +89,16 @@ def __init__(
             self.report.lineage_end_time,
         ) = self._lineage_v1.get_time_window()
 
-        self.known_urns: Set[str] = set()  # will be set later
+    def _is_temp_table(self, name: str) -> bool:
+        return (
+            DatasetUrn.create_from_ids(
+                self.platform,
+                name,
+                env=self.config.env,
+                platform_instance=self.config.platform_instance,
+            ).urn()
+            not in self.known_urns
+        )
 
     def build(
         self,
@@ -107,15 +118,6 @@ def build(
             for schema, tables in schemas.items()
             for table in tables
         }
-        self.aggregator._is_temp_table = (
-            lambda name: DatasetUrn.create_from_ids(
-                self.platform,
-                name,
-                env=self.config.env,
-                platform_instance=self.config.platform_instance,
-            ).urn()
-            not in self.known_urns
-        )
 
         # Handle all the temp tables up front.
         if self.config.resolve_temp_table_in_lineage:
diff --git a/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py b/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py
index c358bee6daae80..e7b0527d30d978 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py
@@ -90,7 +90,9 @@ def schema_count(self) -> int:
             )[0][0]
         )
 
-    def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
+    def get_urn_for_table(
+        self, table: _TableName, lower: bool = False, mixed: bool = False
+    ) -> str:
         # TODO: Validate that this is the correct 2/3 layer hierarchy for the platform.
 
         table_name = ".".join(
@@ -101,7 +103,10 @@ def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str:
 
         if lower:
             table_name = table_name.lower()
-            platform_instance = platform_instance.lower() if platform_instance else None
+            if not mixed:
+                platform_instance = (
+                    platform_instance.lower() if platform_instance else None
+                )
 
         if self.platform == "bigquery":
             # Normalize shard numbers and other BigQuery weirdness.
@@ -131,6 +136,20 @@ def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]:
             if schema_info:
                 return urn_lower, schema_info
 
+        # Our treatment of platform instances when lowercasing urns
+        # is inconsistent. In some places (e.g. Snowflake), we lowercase
+        # the table names but not the platform instance. In other places
+        # (e.g. Databricks), we lowercase everything because it happens
+        # via the automatic lowercasing helper.
+        # See https://github.com/datahub-project/datahub/pull/8928.
+        # While we have this sort of inconsistency, we should also
+        # check the mixed case urn, as a last resort.
+        urn_mixed = self.get_urn_for_table(table, lower=True, mixed=True)
+        if urn_mixed not in {urn, urn_lower}:
+            schema_info = self._resolve_schema_info(urn_mixed)
+            if schema_info:
+                return urn_mixed, schema_info
+
         if self._prefers_urn_lower():
             return urn_lower, None
         else:
diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
index fed1d6b587baa7..036d57c94ee96f 100644
--- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
+++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
@@ -537,8 +537,13 @@ def _maybe_format_query(self, query: str) -> str:
         return query
 
     @functools.lru_cache(maxsize=128)
-    def _name_from_urn(self, urn: UrnStr) -> str:
-        name = DatasetUrn.from_string(urn).name
+    def _name_from_urn(self, urn: UrnStr) -> Optional[str]:
+        urn_obj = DatasetUrn.from_string(urn)
+        if urn_obj.platform != self.platform.urn():
+            # If this is external (e.g. s3), we don't know the name.
+            return None
+
+        name = urn_obj.name
         if (
             platform_instance := self._schema_resolver.platform_instance
         ) and name.startswith(platform_instance):
@@ -549,14 +554,22 @@ def _name_from_urn(self, urn: UrnStr) -> str:
     def is_temp_table(self, urn: UrnStr) -> bool:
         if self._is_temp_table is None:
             return False
-        return self._is_temp_table(self._name_from_urn(urn))
+        name = self._name_from_urn(urn)
+        if name is None:
+            # External tables are not temp tables.
+            return False
+        return self._is_temp_table(name)
 
-    def is_allowed_table(self, urn: UrnStr) -> bool:
+    def is_allowed_table(self, urn: UrnStr, allow_external: bool = True) -> bool:
         if self.is_temp_table(urn):
             return False
         if self._is_allowed_table is None:
             return True
-        return self._is_allowed_table(self._name_from_urn(urn))
+        name = self._name_from_urn(urn)
+        if name is None:
+            # Treat external tables specially.
+            return allow_external
+        return self._is_allowed_table(name)
 
     def add(
         self,
@@ -851,7 +864,7 @@ def add_preparsed_query(
             upstream_fields = parsed.column_usage or {}
             for upstream_urn in parsed.upstreams:
                 # If the upstream table is a temp table or otherwise denied by filters, don't log usage for it.
-                if not self.is_allowed_table(upstream_urn) or (
+                if not self.is_allowed_table(upstream_urn, allow_external=False) or (
                     require_out_table_schema
                     and not self._schema_resolver.has_urn(upstream_urn)
                 ):
diff --git a/metadata-ingestion/src/datahub/upgrade/upgrade.py b/metadata-ingestion/src/datahub/upgrade/upgrade.py
index dd2829ba0d2365..fb14514588e5fc 100644
--- a/metadata-ingestion/src/datahub/upgrade/upgrade.py
+++ b/metadata-ingestion/src/datahub/upgrade/upgrade.py
@@ -13,6 +13,7 @@
 from datahub import __version__
 from datahub.cli.config_utils import load_client_config
 from datahub.ingestion.graph.client import DataHubGraph
+from datahub.utilities.perf_timer import PerfTimer
 
 log = logging.getLogger(__name__)
 
@@ -113,7 +114,7 @@ async def get_server_config(gms_url: str, token: Optional[str]) -> dict:
 
     async with aiohttp.ClientSession() as session:
         config_endpoint = f"{gms_url}/config"
-        async with session.get(config_endpoint) as dh_response:
+        async with session.get(config_endpoint, headers=headers) as dh_response:
             dh_response_json = await dh_response.json()
             return dh_response_json
 
@@ -167,7 +168,28 @@ async def get_server_version_stats(
     return (server_type, server_version, current_server_release_date)
 
 
-async def retrieve_version_stats(
+def retrieve_version_stats(
+    timeout: float, graph: Optional[DataHubGraph] = None
+) -> Optional[DataHubVersionStats]:
+    version_stats: Optional[DataHubVersionStats] = None
+
+    async def _get_version_with_timeout() -> None:
+        # TODO: Once we're on Python 3.11+, replace with asyncio.timeout.
+        stats_future = _retrieve_version_stats(graph)
+
+        try:
+            nonlocal version_stats
+            version_stats = await asyncio.wait_for(stats_future, timeout=timeout)
+        except asyncio.TimeoutError:
+            log.debug("Timed out while fetching version stats")
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(_get_version_with_timeout())
+
+    return version_stats
+
+
+async def _retrieve_version_stats(
     server: Optional[DataHubGraph] = None,
 ) -> Optional[DataHubVersionStats]:
     try:
@@ -263,7 +285,7 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i
         return server.version.micro - client.version.micro
 
 
-def maybe_print_upgrade_message(  # noqa: C901
+def _maybe_print_upgrade_message(  # noqa: C901
     version_stats: Optional[DataHubVersionStats],
 ) -> None:  # noqa: C901
     days_before_cli_stale = 7
@@ -378,28 +400,34 @@ def maybe_print_upgrade_message(  # noqa: C901
             pass
 
 
+def clip(val: float, min_val: float, max_val: float) -> float:
+    return max(min_val, min(val, max_val))
+
+
+def check_upgrade_post(
+    main_method_runtime: float,
+    graph: Optional[DataHubGraph] = None,
+) -> None:
+    # Guarantees: this method will not throw, and will not block for more than 3 seconds.
+
+    version_stats_timeout = clip(main_method_runtime / 10, 0.7, 3.0)
+    try:
+        version_stats = retrieve_version_stats(
+            timeout=version_stats_timeout, graph=graph
+        )
+        _maybe_print_upgrade_message(version_stats=version_stats)
+    except Exception as e:
+        log.debug(f"Failed to check for upgrades due to {e}")
+
+
 def check_upgrade(func: Callable[..., T]) -> Callable[..., T]:
     @wraps(func)
     def async_wrapper(*args: Any, **kwargs: Any) -> Any:
-        async def run_inner_func():
-            return func(*args, **kwargs)
-
-        async def run_func_check_upgrade():
-            version_stats_future = asyncio.ensure_future(retrieve_version_stats())
-            main_func_future = asyncio.ensure_future(run_inner_func())
-            ret = await main_func_future
-
-            # the main future has returned
-            # we check the other futures quickly
-            try:
-                version_stats = await asyncio.wait_for(version_stats_future, 0.5)
-                maybe_print_upgrade_message(version_stats=version_stats)
-            except Exception:
-                log.debug("timed out waiting for version stats to be computed")
+        with PerfTimer() as timer:
+            ret = func(*args, **kwargs)
 
-            return ret
+        check_upgrade_post(main_method_runtime=timer.elapsed_seconds())
 
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(run_func_check_upgrade())
+        return ret
 
     return async_wrapper
diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py
index 92413eeb674f54..237ffc6dc611b9 100644
--- a/metadata-ingestion/src/datahub/utilities/partition_executor.py
+++ b/metadata-ingestion/src/datahub/utilities/partition_executor.py
@@ -294,6 +294,9 @@ def _clearinghouse_worker(self) -> None:  # noqa: C901
         def _handle_batch_completion(
             batch: List[_BatchPartitionWorkItem], future: Future
         ) -> None:
+            nonlocal workers_available
+            workers_available += 1
+
             with clearinghouse_state_lock:
                 for item in batch:
                     keys_no_longer_in_flight.add(item.key)
diff --git a/metadata-ingestion/src/datahub/utilities/perf_timer.py b/metadata-ingestion/src/datahub/utilities/perf_timer.py
index 18384420bfefbd..9488683d6d8cac 100644
--- a/metadata-ingestion/src/datahub/utilities/perf_timer.py
+++ b/metadata-ingestion/src/datahub/utilities/perf_timer.py
@@ -9,7 +9,6 @@
 class PerfTimer(AbstractContextManager):
     """
     A context manager that gives easy access to elapsed time for performance measurement.
-
     """
 
     def __init__(self) -> None:
diff --git a/metadata-ingestion/src/datahub/utilities/threading_timeout.py b/metadata-ingestion/src/datahub/utilities/threading_timeout.py
new file mode 100644
index 00000000000000..e2caf57ad2116a
--- /dev/null
+++ b/metadata-ingestion/src/datahub/utilities/threading_timeout.py
@@ -0,0 +1,42 @@
+import contextlib
+import functools
+import platform
+from typing import ContextManager
+
+from stopit import ThreadingTimeout as _ThreadingTimeout, TimeoutException
+
+__all__ = ["threading_timeout", "TimeoutException"]
+
+
+@functools.lru_cache(maxsize=1)
+def _is_cpython() -> bool:
+    """Check if we're running on CPython."""
+    return platform.python_implementation() == "CPython"
+
+
+def threading_timeout(timeout: float) -> ContextManager[None]:
+    """A timeout context manager that uses stopit's ThreadingTimeout underneath.
+
+    This is only supported on CPython.
+    That's because stopit.ThreadingTimeout uses a CPython-internal method to raise
+    an exception (the timeout error) in another thread. See stopit.threadstop.async_raise.
+
+    Reference: https://github.com/glenfant/stopit
+
+    Args:
+        timeout: The timeout in seconds. If <= 0, no timeout is applied.
+
+    Raises:
+        RuntimeError: If the timeout is not supported on the current Python implementation.
+        TimeoutException: If the timeout is exceeded.
+    """
+
+    if timeout <= 0:
+        return contextlib.nullcontext()
+
+    if not _is_cpython():
+        raise RuntimeError(
+            f"Timeout is only supported on CPython, not {platform.python_implementation()}"
+        )
+
+    return _ThreadingTimeout(timeout, swallow_exc=False)
diff --git a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json
index ad270857dd7fc0..ffa8d43a6a5ab4 100755
--- a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json
+++ b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json
@@ -20,7 +20,7 @@
                     "com.linkedin.pegasus2avro.common.InstitutionalMemory": {
                         "elements": [
                             {
-                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/",
+                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/",
                                 "description": "Link to call for the dataset.",
                                 "createStamp": {
                                     "time": 1586847600,
@@ -71,7 +71,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -95,7 +96,7 @@
                     "com.linkedin.pegasus2avro.common.InstitutionalMemory": {
                         "elements": [
                             {
-                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v2",
+                                "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/v2",
                                 "description": "Link to call for the dataset.",
                                 "createStamp": {
                                     "time": 1586847600,
@@ -146,7 +147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +163,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -176,7 +179,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1586847600000,
-        "runId": "openapi-2020_04_14-07_00_00"
+        "runId": "openapi-2020_04_14-07_00_00-m1k7d5",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml
index e9dc03482c2bca..8575c23239461e 100644
--- a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml
+++ b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml
@@ -2,8 +2,8 @@ source:
   type: openapi
   config:
     name: test_openapi
-    url: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/
-    swagger_file: v3.0/api-with-examples.yaml
+    url: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/
+    swagger_file: v3.0/pass/api-with-examples.yaml
 
 
 sink:
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
index e8be3aa9c0ac73..1039240942a5ee 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json
@@ -8,7 +8,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "demo-workspace"
+                "workspace": "demo-workspace",
+                "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108",
+                "workspace_name": "demo-workspace",
+                "workspace_type": "Workspace"
             },
             "name": "demo-workspace"
         }
@@ -3957,7 +3960,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "second-demo-workspace"
+                "workspace": "second-demo-workspace",
+                "workspace_id": "64ED5CAD-7C22-4684-8180-826122881108",
+                "workspace_name": "second-demo-workspace",
+                "workspace_type": "Workspace"
             },
             "name": "second-demo-workspace"
         }
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
index e134d795af9ef0..0d3a0c0cc6f97b 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json
@@ -8,7 +8,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "64ED5CAD-7C10-4684-8180-826122881108"
+                "workspace": "64ED5CAD-7C10-4684-8180-826122881108",
+                "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108",
+                "workspace_name": "demo-workspace",
+                "workspace_type": "Workspace"
             },
             "name": "demo-workspace"
         }
diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json
index f8c0fdc17c8802..c605f939235cda 100644
--- a/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json
+++ b/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json
@@ -8,7 +8,10 @@
         "json": {
             "customProperties": {
                 "platform": "powerbi",
-                "workspace": "Jane Smith Workspace"
+                "workspace": "Jane Smith Workspace",
+                "workspace_id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C",
+                "workspace_name": "Jane Smith Workspace",
+                "workspace_type": "PersonalGroup"
             },
             "name": "Jane Smith Workspace"
         }
diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py b/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
index 5a33034f274dc2..e5fa980bec4522 100644
--- a/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
+++ b/metadata-ingestion/tests/unit/sql_parsing/test_schemaresolver.py
@@ -41,6 +41,11 @@ def test_get_urn_for_table_lowercase():
         == "urn:li:dataset:(urn:li:dataPlatform:mssql,uppercased-instance.database.dataset.table,PROD)"
     )
 
+    assert (
+        schema_resolver.get_urn_for_table(table=table, lower=True, mixed=True)
+        == "urn:li:dataset:(urn:li:dataPlatform:mssql,Uppercased-Instance.database.dataset.table,PROD)"
+    )
+
 
 def test_get_urn_for_table_not_lower_should_keep_capital_letters():
     schema_resolver = SchemaResolver(
diff --git a/metadata-ingestion/tests/unit/test_datahub_source.py b/metadata-ingestion/tests/unit/test_datahub_source.py
index adc131362b326b..67b2b85d9af6dd 100644
--- a/metadata-ingestion/tests/unit/test_datahub_source.py
+++ b/metadata-ingestion/tests/unit/test_datahub_source.py
@@ -1,51 +1,41 @@
-from dataclasses import dataclass
+from typing import Any, Dict
 
 import pytest
 
-from datahub.ingestion.source.datahub.datahub_database_reader import (
-    VersionOrderable,
-    VersionOrderer,
-)
-
-
-@dataclass
-class MockRow(VersionOrderable):
-    createdon: int
-    version: int
-    urn: str
+from datahub.ingestion.source.datahub.datahub_database_reader import VersionOrderer
 
 
 @pytest.fixture
 def rows():
     return [
-        MockRow(0, 0, "one"),
-        MockRow(0, 1, "one"),
-        MockRow(0, 0, "two"),
-        MockRow(0, 0, "three"),
-        MockRow(0, 1, "three"),
-        MockRow(0, 2, "three"),
-        MockRow(0, 1, "two"),
-        MockRow(0, 4, "three"),
-        MockRow(0, 5, "three"),
-        MockRow(1, 6, "three"),
-        MockRow(1, 0, "four"),
-        MockRow(2, 0, "five"),
-        MockRow(2, 1, "six"),
-        MockRow(2, 0, "six"),
-        MockRow(3, 0, "seven"),
-        MockRow(3, 0, "eight"),
+        {"createdon": 0, "version": 0, "urn": "one"},
+        {"createdon": 0, "version": 1, "urn": "one"},
+        {"createdon": 0, "version": 0, "urn": "two"},
+        {"createdon": 0, "version": 0, "urn": "three"},
+        {"createdon": 0, "version": 1, "urn": "three"},
+        {"createdon": 0, "version": 2, "urn": "three"},
+        {"createdon": 0, "version": 1, "urn": "two"},
+        {"createdon": 0, "version": 4, "urn": "three"},
+        {"createdon": 0, "version": 5, "urn": "three"},
+        {"createdon": 1, "version": 6, "urn": "three"},
+        {"createdon": 1, "version": 0, "urn": "four"},
+        {"createdon": 2, "version": 0, "urn": "five"},
+        {"createdon": 2, "version": 1, "urn": "six"},
+        {"createdon": 2, "version": 0, "urn": "six"},
+        {"createdon": 3, "version": 0, "urn": "seven"},
+        {"createdon": 3, "version": 0, "urn": "eight"},
     ]
 
 
 def test_version_orderer(rows):
-    orderer = VersionOrderer[MockRow](enabled=True)
+    orderer = VersionOrderer[Dict[str, Any]](enabled=True)
     ordered_rows = list(orderer(rows))
     assert ordered_rows == sorted(
-        ordered_rows, key=lambda x: (x.createdon, x.version == 0)
+        ordered_rows, key=lambda x: (x["createdon"], x["version"] == 0)
     )
 
 
 def test_version_orderer_disabled(rows):
-    orderer = VersionOrderer[MockRow](enabled=False)
+    orderer = VersionOrderer[Dict[str, Any]](enabled=False)
     ordered_rows = list(orderer(rows))
     assert ordered_rows == rows
diff --git a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
index 1de76a32fb708a..166e40ef543087 100644
--- a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
+++ b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
@@ -8,7 +8,7 @@
 approx = partial(pytest.approx, rel=2e-2)
 
 
-def test_perf_timer_simple():
+def test_perf_timer_simple() -> None:
     with PerfTimer() as timer:
         time.sleep(0.4)
         assert approx(timer.elapsed_seconds()) == 0.4
@@ -16,7 +16,7 @@ def test_perf_timer_simple():
     assert approx(timer.elapsed_seconds()) == 0.4
 
 
-def test_perf_timer_paused_timer():
+def test_perf_timer_paused_timer() -> None:
     with PerfTimer() as current_timer:
         time.sleep(0.5)
         assert approx(current_timer.elapsed_seconds()) == 0.5
@@ -29,7 +29,7 @@ def test_perf_timer_paused_timer():
     assert approx(current_timer.elapsed_seconds()) == 0.7
 
 
-def test_generator_with_paused_timer():
+def test_generator_with_paused_timer() -> None:
     n = 4
 
     def generator_function():
@@ -46,3 +46,15 @@ def generator_function():
         seq = generator_function()
         list([i for i in seq])
         assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * n + 0.2 * n
+
+
+def test_perf_timer_reuse() -> None:
+    timer = PerfTimer()
+
+    with timer:
+        time.sleep(0.2)
+
+    with timer:
+        time.sleep(0.3)
+
+    assert approx(timer.elapsed_seconds()) == 0.5
diff --git a/metadata-ingestion/tests/unit/utilities/test_threading_timeout.py b/metadata-ingestion/tests/unit/utilities/test_threading_timeout.py
new file mode 100644
index 00000000000000..c52d18bdd55c2b
--- /dev/null
+++ b/metadata-ingestion/tests/unit/utilities/test_threading_timeout.py
@@ -0,0 +1,31 @@
+import time
+
+import pytest
+
+from datahub.utilities.threading_timeout import TimeoutException, threading_timeout
+
+
+def test_timeout_no_timeout():
+    # Should complete without raising an exception
+    with threading_timeout(1.0):
+        time.sleep(0.1)
+
+
+def test_timeout_raises():
+    # Should raise TimeoutException
+    with pytest.raises(TimeoutException):
+        with threading_timeout(0.1):
+            time.sleep(0.5)
+
+
+def test_timeout_early_exit():
+    # Test that context manager handles other exceptions properly
+    with pytest.raises(ValueError):
+        with threading_timeout(1.0):
+            raise ValueError("Early exit")
+
+
+def test_timeout_zero():
+    # Should not raise an exception
+    with threading_timeout(0.0):
+        pass
diff --git a/metadata-integration/java/acryl-spark-lineage/README.md b/metadata-integration/java/acryl-spark-lineage/README.md
index 9caa5a6dec65db..bd0a58b635b483 100644
--- a/metadata-integration/java/acryl-spark-lineage/README.md
+++ b/metadata-integration/java/acryl-spark-lineage/README.md
@@ -184,7 +184,7 @@ information like tokens.
 | spark.datahub.platform.s3.path_spec_list               |          |                       | List of pathspec per platform                                                                                                                                                             |
 | spark.datahub.metadata.dataset.include_schema_metadata | false    |                       | Emit dataset schema metadata based on the spark execution. It is recommended to get schema information from platform specific DataHub sources  as this is less reliable                   |
 | spark.datahub.flow_name                                |          |                       | If it is set it will be used as the DataFlow name otherwise it uses spark app name as flow_name                                                                                           |
-| spark.datahub.partition_regexp_pattern                 |          |                       | Strip partition part from the path if path end matches with the specified regexp. Example `year=.*/month=.*/day=.*`                                                                       |
+| spark.datahub.file_partition_regexp                    |          |                       | Strip partition part from the path if path end matches with the specified regexp. Example `year=.*/month=.*/day=.*`                                                                       |
 | spark.datahub.tags                                     |          |                       | Comma separated list of tags to attach to the DataFlow                                                                                                                                    |
 | spark.datahub.domains                                  |          |                       | Comma separated list of domain urns to attach to the DataFlow                                                                                                                             |
 | spark.datahub.stage_metadata_coalescing                |          |                       | Normally it coalesces and sends metadata at the onApplicationEnd event which is never called on Databricks or on Glue. You should enable this on Databricks if you want coalesced run.    |
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
index 7a5fdeaeb8e0d9..0bcc7db9e87408 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
@@ -82,7 +82,10 @@ private Optional<Emitter> getEmitter() {
             (KafkaDatahubEmitterConfig) datahubConf.getDatahubEmitterConfig();
         try {
           emitter =
-              Optional.of(new KafkaEmitter(datahubKafkaEmitterConfig.getKafkaEmitterConfig()));
+              Optional.of(
+                  new KafkaEmitter(
+                      datahubKafkaEmitterConfig.getKafkaEmitterConfig(),
+                      datahubKafkaEmitterConfig.getMcpTopic()));
         } catch (IOException e) {
           throw new RuntimeException(e);
         }
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
index 52507a682a1f8b..ee0938edb50454 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
@@ -188,8 +188,12 @@ public Optional<DatahubEmitterConfig> initializeEmitter(Config sparkConf) {
                   });
           kafkaEmitterConfig.producerConfig(kafkaConfig);
         }
-
-        return Optional.of(new KafkaDatahubEmitterConfig(kafkaEmitterConfig.build()));
+        if (sparkConf.hasPath(SparkConfigParser.KAFKA_MCP_TOPIC)) {
+          String mcpTopic = sparkConf.getString(SparkConfigParser.KAFKA_MCP_TOPIC);
+          return Optional.of(new KafkaDatahubEmitterConfig(kafkaEmitterConfig.build(), mcpTopic));
+        } else {
+          return Optional.of(new KafkaDatahubEmitterConfig(kafkaEmitterConfig.build()));
+        }
       case "file":
         log.info("File Emitter Configuration: File emitter will be used");
         FileEmitterConfig.FileEmitterConfigBuilder fileEmitterConfig = FileEmitterConfig.builder();
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java
index 6ed66dbc9230f3..a5f9b59f708468 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/KafkaDatahubEmitterConfig.java
@@ -1,5 +1,6 @@
 package datahub.spark.conf;
 
+import datahub.client.kafka.KafkaEmitter;
 import datahub.client.kafka.KafkaEmitterConfig;
 import lombok.Getter;
 import lombok.Setter;
@@ -11,8 +12,15 @@
 public class KafkaDatahubEmitterConfig implements DatahubEmitterConfig {
   final String type = "kafka";
   KafkaEmitterConfig kafkaEmitterConfig;
+  String mcpTopic;
 
   public KafkaDatahubEmitterConfig(KafkaEmitterConfig kafkaEmitterConfig) {
     this.kafkaEmitterConfig = kafkaEmitterConfig;
+    this.mcpTopic = KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC;
+  }
+
+  public KafkaDatahubEmitterConfig(KafkaEmitterConfig kafkaEmitterConfig, String mcpTopic) {
+    this.kafkaEmitterConfig = kafkaEmitterConfig;
+    this.mcpTopic = mcpTopic;
   }
 }
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
index 630f10b08b411d..45ec5365d09b36 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
@@ -32,6 +32,7 @@ public class SparkConfigParser {
   public static final String DISABLE_SSL_VERIFICATION_KEY = "rest.disable_ssl_verification";
   public static final String MAX_RETRIES = "rest.max_retries";
   public static final String RETRY_INTERVAL_IN_SEC = "rest.retry_interval_in_sec";
+  public static final String KAFKA_MCP_TOPIC = "kafka.mcp_topic";
   public static final String KAFKA_EMITTER_BOOTSTRAP = "kafka.bootstrap";
   public static final String KAFKA_EMITTER_SCHEMA_REGISTRY_URL = "kafka.schema_registry_url";
   public static final String KAFKA_EMITTER_SCHEMA_REGISTRY_CONFIG = "kafka.schema_registry_config";
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
index 0d0341562e7dd0..e8ceeab6963218 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java
@@ -9,7 +9,7 @@
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 
-class AvroSerializer {
+public class AvroSerializer {
 
   private final Schema _recordSchema;
   private final Schema _genericAspectSchema;
diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java
index d00dc09669045f..777d2d5f301d75 100644
--- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java
+++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java
@@ -31,14 +31,25 @@ public class KafkaEmitter implements Emitter {
   private final Properties kafkaConfigProperties;
   private AvroSerializer _avroSerializer;
   private static final int ADMIN_CLIENT_TIMEOUT_MS = 5000;
+  private final String mcpKafkaTopic;
 
   /**
    * The default constructor
    *
-   * @param config
-   * @throws IOException
+   * @param config KafkaEmitterConfig
+   * @throws IOException when Avro Serialization fails
    */
   public KafkaEmitter(KafkaEmitterConfig config) throws IOException {
+    this(config, DEFAULT_MCP_KAFKA_TOPIC);
+  }
+
+  /**
+   * Constructor that takes in KafkaEmitterConfig and mcp Kafka Topic Name
+   *
+   * @param config KafkaEmitterConfig
+   * @throws IOException when Avro Serialization fails
+   */
+  public KafkaEmitter(KafkaEmitterConfig config, String mcpKafkaTopic) throws IOException {
     this.config = config;
     kafkaConfigProperties = new Properties();
     kafkaConfigProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.config.getBootstrap());
@@ -54,6 +65,7 @@ public KafkaEmitter(KafkaEmitterConfig config) throws IOException {
 
     producer = new KafkaProducer<>(kafkaConfigProperties);
     _avroSerializer = new AvroSerializer();
+    this.mcpKafkaTopic = mcpKafkaTopic;
   }
 
   @Override
@@ -73,8 +85,7 @@ public Future<MetadataWriteResponse> emit(MetadataChangeProposal mcp, Callback d
       throws IOException {
     GenericRecord genricRecord = _avroSerializer.serialize(mcp);
     ProducerRecord<Object, Object> record =
-        new ProducerRecord<>(
-            KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC, mcp.getEntityUrn().toString(), genricRecord);
+        new ProducerRecord<>(this.mcpKafkaTopic, mcp.getEntityUrn().toString(), genricRecord);
     org.apache.kafka.clients.producer.Callback callback =
         new org.apache.kafka.clients.producer.Callback() {
 
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java
index 1381e9560b7e53..cef463802a6b14 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java
@@ -44,6 +44,8 @@ public class AggregationQueryBuilderTest {
   public void setup() throws RemoteInvocationException, URISyntaxException {
     Urn helloUrn = Urn.createFromString("urn:li:structuredProperty:hello");
     Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten");
+    Urn underscoresAndDotsUrn =
+        Urn.createFromString("urn:li:structuredProperty:under.scores.and.dots_make_a_mess");
 
     // legacy
     aspectRetriever = mock(AspectRetriever.class);
@@ -75,6 +77,21 @@ public void setup() throws RemoteInvocationException, URISyntaxException {
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinition.data()))));
 
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinition =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL);
+    structPropUnderscoresAndDotsDefinition.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinition.setQualifiedName("under.scores.and.dots_make_a_mess");
+    structPropUnderscoresAndDotsDefinition.setDisplayName("under.scores.and.dots_make_a_mess");
+    when(aspectRetriever.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinition.data()))));
+
     // V1
     aspectRetrieverV1 = mock(AspectRetriever.class);
     when(aspectRetrieverV1.getEntityRegistry())
@@ -105,6 +122,21 @@ public void setup() throws RemoteInvocationException, URISyntaxException {
                 Map.of(
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinitionV1.data()))));
+
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinitionV1 =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinitionV1.setVersion("00000000000001");
+    structPropUnderscoresAndDotsDefinitionV1.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinitionV1.setQualifiedName("under.scores.and.dots_make_a_mess");
+    structPropUnderscoresAndDotsDefinitionV1.setDisplayName("under.scores.and.dots_make_a_mess");
+    when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinitionV1.data()))));
   }
 
   @Test
@@ -269,6 +301,46 @@ public void testAggregateOverStructuredProperty() {
         Set.of("structuredProperties.ab_fgh_ten.keyword", "structuredProperties.hello.keyword"));
   }
 
+  @Test
+  public void testAggregateOverStructuredPropertyNamespaced() {
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    List<AggregationBuilder> aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever),
+            List.of("structuredProperties.under.scores.and.dots_make_a_mess"));
+    Assert.assertEquals(aggs.size(), 1);
+    AggregationBuilder aggBuilder = aggs.get(0);
+    Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder);
+    TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder;
+    // Check that field name is sanitized to correct field name
+    Assert.assertEquals(
+        agg.field(),
+        "structuredProperties.under_scores_and_dots_make_a_mess.keyword",
+        "Terms aggregate must be on a keyword or subfield keyword");
+
+    // Two structured properties
+    aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever),
+            List.of(
+                "structuredProperties.under.scores.and.dots_make_a_mess",
+                "structuredProperties.hello"));
+    Assert.assertEquals(aggs.size(), 2);
+    Assert.assertEquals(
+        aggs.stream()
+            .map(aggr -> ((TermsAggregationBuilder) aggr).field())
+            .collect(Collectors.toSet()),
+        Set.of(
+            "structuredProperties.under_scores_and_dots_make_a_mess.keyword",
+            "structuredProperties.hello.keyword"));
+  }
+
   @Test
   public void testAggregateOverStructuredPropertyV1() {
     SearchConfiguration config = new SearchConfiguration();
@@ -309,6 +381,46 @@ public void testAggregateOverStructuredPropertyV1() {
             "structuredProperties._versioned.hello.00000000000001.string.keyword"));
   }
 
+  @Test
+  public void testAggregateOverStructuredPropertyNamespacedV1() {
+    SearchConfiguration config = new SearchConfiguration();
+    config.setMaxTermBucketSize(25);
+
+    AggregationQueryBuilder builder =
+        new AggregationQueryBuilder(
+            config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of()));
+
+    List<AggregationBuilder> aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1),
+            List.of("structuredProperties.under.scores.and.dots_make_a_mess"));
+    Assert.assertEquals(aggs.size(), 1);
+    AggregationBuilder aggBuilder = aggs.get(0);
+    Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder);
+    TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder;
+    // Check that field name is sanitized to correct field name
+    Assert.assertEquals(
+        agg.field(),
+        "structuredProperties._versioned.under_scores_and_dots_make_a_mess.00000000000001.string.keyword",
+        "Terms aggregation must be on a keyword field or subfield.");
+
+    // Two structured properties
+    aggs =
+        builder.getAggregations(
+            TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1),
+            List.of(
+                "structuredProperties.under.scores.and.dots_make_a_mess",
+                "structuredProperties._versioned.hello.00000000000001.string"));
+    Assert.assertEquals(aggs.size(), 2);
+    Assert.assertEquals(
+        aggs.stream()
+            .map(aggr -> ((TermsAggregationBuilder) aggr).field())
+            .collect(Collectors.toSet()),
+        Set.of(
+            "structuredProperties._versioned.under_scores_and_dots_make_a_mess.00000000000001.string.keyword",
+            "structuredProperties._versioned.hello.00000000000001.string.keyword"));
+  }
+
   @Test
   public void testAggregateOverFieldsAndStructProp() {
     SearchableAnnotation annotation1 =
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
index a3ef62760d7972..718a00d067ce5a 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java
@@ -1,24 +1,34 @@
 package com.linkedin.metadata.search.query.request;
 
 import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES;
+import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
 import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion;
 import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion;
 import static com.linkedin.metadata.utils.CriterionUtils.buildIsNullCriterion;
 import static com.linkedin.metadata.utils.SearchUtil.*;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import static org.testng.Assert.*;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.data.DataMap;
 import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
+import com.linkedin.entity.Aspect;
 import com.linkedin.metadata.TestEntitySpecBuilder;
+import com.linkedin.metadata.aspect.AspectRetriever;
+import com.linkedin.metadata.aspect.GraphRetriever;
 import com.linkedin.metadata.config.search.ExactMatchConfiguration;
 import com.linkedin.metadata.config.search.PartialConfiguration;
 import com.linkedin.metadata.config.search.SearchConfiguration;
 import com.linkedin.metadata.config.search.WordGramConfiguration;
+import com.linkedin.metadata.entity.SearchRetriever;
 import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.models.StructuredPropertyUtils;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
@@ -28,6 +38,8 @@
 import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain;
 import com.linkedin.metadata.search.elasticsearch.query.request.SearchRequestHandler;
 import io.datahubproject.metadata.context.OperationContext;
+import io.datahubproject.metadata.context.RetrieverContext;
+import io.datahubproject.test.metadata.context.TestOperationContexts;
 import io.datahubproject.test.search.config.SearchCommonTestConfiguration;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -633,6 +645,49 @@ public void testBrowsePathQueryFilter() {
     assertEquals(((ExistsQueryBuilder) mustHaveV1.must().get(0)).fieldName(), "browsePaths");
   }
 
+  @Test(expectedExceptions = IllegalArgumentException.class)
+  public void testInvalidStructuredProperty() {
+    AspectRetriever aspectRetriever = mock(AspectRetriever.class);
+    Map<Urn, Map<String, Aspect>> aspectResponse = new HashMap<>();
+    DataMap statusData = new DataMap();
+    statusData.put("removed", true);
+    Aspect status = new Aspect(statusData);
+    Urn structPropUrn = StructuredPropertyUtils.toURNFromFQN("under.scores.and.dots.make_a_mess");
+    aspectResponse.put(structPropUrn, ImmutableMap.of(STATUS_ASPECT_NAME, status));
+    when(aspectRetriever.getLatestAspectObjects(
+            Collections.singleton(structPropUrn), ImmutableSet.of(STATUS_ASPECT_NAME)))
+        .thenReturn(aspectResponse);
+    OperationContext mockRetrieverContext =
+        TestOperationContexts.systemContextNoSearchAuthorization(
+            RetrieverContext.builder()
+                .aspectRetriever(aspectRetriever)
+                .graphRetriever(mock(GraphRetriever.class))
+                .searchRetriever(mock(SearchRetriever.class))
+                .build());
+
+    Criterion structuredPropCriterion =
+        buildExistsCriterion("structuredProperties.under.scores.and.dots.make_a_mess");
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(structuredPropCriterion);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+
+    BoolQueryBuilder test =
+        SearchRequestHandler.getFilterQuery(
+            mockRetrieverContext.withSearchFlags(flags -> flags.setFulltext(false)),
+            filter,
+            new HashMap<>(),
+            QueryFilterRewriteChain.EMPTY);
+  }
+
   @Test
   public void testQueryByDefault() {
     final Set<String> COMMON =
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java
index 6665faacae3373..892f7088e7f614 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java
@@ -43,6 +43,8 @@ public class ESUtilsTest {
   @BeforeClass
   public static void setup() throws RemoteInvocationException, URISyntaxException {
     Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten");
+    Urn underscoresAndDotsUrn =
+        Urn.createFromString("urn:li:structuredProperty:under.scores.and.dots_make_a_mess");
 
     // legacy
     aspectRetriever = mock(AspectRetriever.class);
@@ -62,6 +64,20 @@ public static void setup() throws RemoteInvocationException, URISyntaxException
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinition.data()))));
 
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinition =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL);
+    structPropUnderscoresAndDotsDefinition.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinition.setQualifiedName("under.scores.and.dots_make_a_mess");
+    when(aspectRetriever.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinition.data()))));
+
     // V1
     aspectRetrieverV1 = mock(AspectRetriever.class);
     when(aspectRetrieverV1.getEntityRegistry())
@@ -80,6 +96,20 @@ public static void setup() throws RemoteInvocationException, URISyntaxException
                 Map.of(
                     STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
                     new Aspect(structPropAbFghTenDefinitionV1.data()))));
+
+    StructuredPropertyDefinition structPropUnderscoresAndDotsDefinitionV1 =
+        new StructuredPropertyDefinition();
+    structPropUnderscoresAndDotsDefinitionV1.setVersion("00000000000001");
+    structPropUnderscoresAndDotsDefinitionV1.setValueType(
+        Urn.createFromString(DATA_TYPE_URN_PREFIX + "string"));
+    structPropUnderscoresAndDotsDefinitionV1.setQualifiedName("under.scores.and.dots_make_a_mess");
+    when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(underscoresAndDotsUrn)), anySet()))
+        .thenReturn(
+            Map.of(
+                underscoresAndDotsUrn,
+                Map.of(
+                    STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME,
+                    new Aspect(structPropUnderscoresAndDotsDefinitionV1.data()))));
   }
 
   @Test
@@ -703,6 +733,31 @@ public void testGetQueryBuilderFromStructPropEqualsValue() {
     Assert.assertEquals(result.toString(), expected);
   }
 
+  @Test
+  public void testGetQueryBuilderFromNamespacedStructPropEqualsValue() {
+
+    final Criterion singleValueCriterion =
+        buildCriterion(
+            "structuredProperties.under.scores.and.dots_make_a_mess", Condition.EQUAL, "value1");
+
+    OperationContext opContext = mock(OperationContext.class);
+    when(opContext.getAspectRetriever()).thenReturn(aspectRetriever);
+    QueryBuilder result =
+        ESUtils.getQueryBuilderFromCriterion(
+            singleValueCriterion, false, new HashMap<>(), opContext, QueryFilterRewriteChain.EMPTY);
+    String expected =
+        "{\n"
+            + "  \"terms\" : {\n"
+            + "    \"structuredProperties.under_scores_and_dots_make_a_mess.keyword\" : [\n"
+            + "      \"value1\"\n"
+            + "    ],\n"
+            + "    \"boost\" : 1.0,\n"
+            + "    \"_name\" : \"structuredProperties.under.scores.and.dots_make_a_mess\"\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(result.toString(), expected);
+  }
+
   @Test
   public void testGetQueryBuilderFromStructPropEqualsValueV1() {
 
@@ -731,6 +786,35 @@ public void testGetQueryBuilderFromStructPropEqualsValueV1() {
     Assert.assertEquals(result.toString(), expected);
   }
 
+  @Test
+  public void testGetQueryBuilderFromNamespacedStructPropEqualsValueV1() {
+
+    final Criterion singleValueCriterion =
+        buildCriterion(
+            "structuredProperties.under.scores.and.dots_make_a_mess", Condition.EQUAL, "value1");
+
+    OperationContext opContextV1 = mock(OperationContext.class);
+    when(opContextV1.getAspectRetriever()).thenReturn(aspectRetrieverV1);
+    QueryBuilder result =
+        ESUtils.getQueryBuilderFromCriterion(
+            singleValueCriterion,
+            false,
+            new HashMap<>(),
+            opContextV1,
+            QueryFilterRewriteChain.EMPTY);
+    String expected =
+        "{\n"
+            + "  \"terms\" : {\n"
+            + "    \"structuredProperties._versioned.under_scores_and_dots_make_a_mess.00000000000001.string.keyword\" : [\n"
+            + "      \"value1\"\n"
+            + "    ],\n"
+            + "    \"boost\" : 1.0,\n"
+            + "    \"_name\" : \"structuredProperties.under.scores.and.dots_make_a_mess\"\n"
+            + "  }\n"
+            + "}";
+    Assert.assertEquals(result.toString(), expected);
+  }
+
   @Test
   public void testGetQueryBuilderFromStructPropExists() {
     final Criterion singleValueCriterion = buildExistsCriterion("structuredProperties.ab.fgh.ten");
diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml
index 10ae176b2c31e2..3ce335fc667e71 100644
--- a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml
+++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml
@@ -38,7 +38,7 @@ bootstrap:
 
     # Ingestion Recipes
     - name: ingestion-datahub-gc
-      version: v1
-      optional: true
+      version: v3
+      optional: false
       mcps_location: "bootstrap_mcps/ingestion-datahub-gc.yaml"
-      values_env: "DATAHUB_GC_BOOTSTRAP_VALUES"
\ No newline at end of file
+      values_env: "DATAHUB_GC_BOOTSTRAP_VALUES"
diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml
index e70ab1162a3816..2c99d0d884ddf7 100644
--- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml
+++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml
@@ -27,6 +27,12 @@
               keep_last_n: {{dataprocess_cleanup.keep_last_n}}{{^dataprocess_cleanup.keep_last_n}}5{{/dataprocess_cleanup.keep_last_n}}
             soft_deleted_entities_cleanup:
               retention_days: {{soft_deleted_entities_cleanup.retention_days}}{{^soft_deleted_entities_cleanup.retention_days}}10{{/soft_deleted_entities_cleanup.retention_days}}
+            execution_request_cleanup:
+              keep_history_min_count: {{execution_request_cleanup.keep_history_min_count}}{{^execution_request_cleanup.keep_history_min_count}}10{{/execution_request_cleanup.keep_history_min_count}}
+              keep_history_max_count: {{execution_request_cleanup.keep_history_max_count}}{{^execution_request_cleanup.keep_history_max_count}}1000{{/execution_request_cleanup.keep_history_max_count}}
+              keep_history_max_days: {{execution_request_cleanup.keep_history_max_days}}{{^execution_request_cleanup.keep_history_max_days}}30{{/execution_request_cleanup.keep_history_max_days}}
+              batch_read_size: {{execution_request_cleanup.batch_read_size}}{{^execution_request_cleanup.batch_read_size}}100{{/execution_request_cleanup.batch_read_size}}
+              enabled: {{execution_request_cleanup.enabled}}{{^execution_request_cleanup.enabled}}false{{/execution_request_cleanup.enabled}}
       extraArgs: {}
       debugMode: false
       executorId: default
diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py
index 8b6fead789b3f7..f0509aad77625f 100644
--- a/smoke-test/tests/structured_properties/test_structured_properties.py
+++ b/smoke-test/tests/structured_properties/test_structured_properties.py
@@ -165,13 +165,13 @@ def get_property_from_entity(
     return None
 
 
-def to_es_name(property_name=None, namespace=default_namespace, qualified_name=None):
+def to_es_filter_name(
+    property_name=None, namespace=default_namespace, qualified_name=None
+):
     if property_name:
-        namespace_field = namespace.replace(".", "_")
-        return f"structuredProperties.{namespace_field}_{property_name}"
+        return f"structuredProperties.{namespace}.{property_name}"
     else:
-        escaped_qualified_name = qualified_name.replace(".", "_")
-        return f"structuredProperties.{escaped_qualified_name}"
+        return f"structuredProperties.{qualified_name}"
 
 
 # @tenacity.retry(
@@ -446,7 +446,7 @@ def test_structured_property_search(
         graph_client.get_urns_by_filter(
             extraFilters=[
                 {
-                    "field": to_es_name(dataset_property_name),
+                    "field": to_es_filter_name(dataset_property_name),
                     "negated": "false",
                     "condition": "EXISTS",
                 }
@@ -475,7 +475,7 @@ def test_structured_property_search(
             entity_types=["tag"],
             extraFilters=[
                 {
-                    "field": to_es_name(
+                    "field": to_es_filter_name(
                         field_property_name, namespace="io.datahubproject.test"
                     ),
                     "negated": "false",
@@ -492,7 +492,7 @@ def test_structured_property_search(
             entity_types=["dataset", "tag"],
             extraFilters=[
                 {
-                    "field": to_es_name(dataset_property_name),
+                    "field": to_es_filter_name(dataset_property_name),
                     "negated": "false",
                     "condition": "EXISTS",
                 }
@@ -690,7 +690,7 @@ def test_dataset_structured_property_soft_delete_search_filter_validation(
         graph_client.get_urns_by_filter(
             extraFilters=[
                 {
-                    "field": to_es_name(dataset_property_name),
+                    "field": to_es_filter_name(property_name=dataset_property_name),
                     "negated": "false",
                     "condition": "EXISTS",
                 }
@@ -710,7 +710,7 @@ def test_dataset_structured_property_soft_delete_search_filter_validation(
             graph_client.get_urns_by_filter(
                 extraFilters=[
                     {
-                        "field": to_es_name(dataset_property_name),
+                        "field": to_es_filter_name(property_name=dataset_property_name),
                         "negated": "false",
                         "condition": "EXISTS",
                     }
@@ -779,7 +779,7 @@ def validate_search(qualified_name, expected):
             graph_client.get_urns_by_filter(
                 extraFilters=[
                     {
-                        "field": to_es_name(qualified_name=qualified_name),
+                        "field": to_es_filter_name(qualified_name=qualified_name),
                         "negated": "false",
                         "condition": "EXISTS",
                     }