From ffda9b1c3e80db5417613a14ad1ebda4dc32d7d4 Mon Sep 17 00:00:00 2001 From: Marco polo Date: Tue, 10 Oct 2023 10:43:15 -0400 Subject: [PATCH] [rfc] Add back "materialize changed and missing" as a dialog (#17115) ## Summary & Motivation We no longer fetch all of the asset status information in the graph up front so we can't show a number on the option, instead we will lazily fetch the stale status on click in a dialog: Screenshot 2023-10-10 at 10 11 42 AM Screenshot 2023-10-10 at 9 00 10 AM Screenshot 2023-10-10 at 10 19 22 AM Screenshot 2023-10-10 at 8 50 23 AM ## How I Tested These Changes Locally --- .../ui-core/src/assets/AssetNodeLineage.tsx | 14 +- .../packages/ui-core/src/assets/AssetView.tsx | 1 - .../CalculateChangedAndMissingDialog.tsx | 219 ++++++++++++++++++ .../src/assets/LaunchAssetExecutionButton.tsx | 51 ++-- .../packages/ui-core/src/assets/Stale.tsx | 4 +- .../CalculateChangedAndMissingDialog.types.ts | 17 ++ 6 files changed, 264 insertions(+), 42 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/CalculateChangedAndMissingDialog.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/types/CalculateChangedAndMissingDialog.types.ts diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx index d2b9bf75793e2..06297f9162030 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx @@ -10,7 +10,7 @@ import { import * as React from 'react'; import styled from 'styled-components'; -import {GraphData, LiveData} from '../asset-graph/Utils'; +import {GraphData} from '../asset-graph/Utils'; import {AssetGraphQueryItem, calculateGraphDistances} from '../asset-graph/useAssetGraphData'; import {AssetKeyInput} from '../graphql/types'; @@ -23,18 +23,9 @@ export const AssetNodeLineage: React.FC<{ setParams: (params: AssetViewParams) => void; assetKey: AssetKeyInput; assetGraphData: GraphData; - liveDataByNode: LiveData; requestedDepth: number; graphQueryItems: AssetGraphQueryItem[]; -}> = ({ - params, - setParams, - assetKey, - liveDataByNode, - assetGraphData, - graphQueryItems, - requestedDepth, -}) => { +}> = ({params, setParams, assetKey, assetGraphData, graphQueryItems, requestedDepth}) => { const maxDistances = React.useMemo( () => calculateGraphDistances(graphQueryItems, assetKey), [graphQueryItems, assetKey], @@ -76,7 +67,6 @@ export const AssetNodeLineage: React.FC<{ {Object.values(assetGraphData.nodes).length > 1 ? ( n.definition)}} /> ) : ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index 6f129498839a3..aee48150e77d6 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -136,7 +136,6 @@ export const AssetView = ({assetKey}: Props) => { params={params} setParams={setParams} assetKey={assetKey} - liveDataByNode={liveDataByNode} requestedDepth={visible.requestedDepth} assetGraphData={visibleAssetGraph.assetGraphData} graphQueryItems={visibleAssetGraph.graphQueryItems} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/CalculateChangedAndMissingDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/CalculateChangedAndMissingDialog.tsx new file mode 100644 index 0000000000000..dae7c8b820a71 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/CalculateChangedAndMissingDialog.tsx @@ -0,0 +1,219 @@ +import {useQuery, gql} from '@apollo/client'; +import { + Spinner, + Dialog, + DialogBody, + DialogFooter, + Button, + Box, + Icon, + Colors, + Checkbox, + MiddleTruncate, +} from '@dagster-io/ui-components'; +import {useVirtualizer} from '@tanstack/react-virtual'; +import React from 'react'; +import {Link} from 'react-router-dom'; +import styled from 'styled-components'; + +import {showCustomAlert} from '../app/CustomAlertProvider'; +import {displayNameForAssetKey} from '../asset-graph/Utils'; +import {Container, Inner, Row} from '../ui/VirtualizedTable'; + +import {isAssetStale, isAssetMissing} from './Stale'; +import {asAssetKeyInput} from './asInput'; +import {assetDetailsPathForKey} from './assetDetailsPathForKey'; +import {AssetKey} from './types'; +import { + AssetStaleStatusQuery, + AssetStaleStatusQueryVariables, +} from './types/CalculateChangedAndMissingDialog.types'; + +export const CalculateChangedAndMissingDialog = React.memo( + ({ + isOpen, + onClose, + assets, + onMaterializeAssets, + }: { + isOpen: boolean; + assets: { + assetKey: AssetKey; + }[]; + onClose: () => void; + onMaterializeAssets: (assets: AssetKey[], e: React.MouseEvent) => void; + }) => { + const {data, loading, error} = useQuery( + ASSET_STALE_STATUS_QUERY, + { + variables: { + assetKeys: assets.map(asAssetKeyInput), + }, + skip: !isOpen, + }, + ); + + const staleOrMissing = React.useMemo( + () => + data?.assetNodes + .filter((node) => isAssetStale(node) || isAssetMissing(node)) + .map(asAssetKeyInput), + [data], + ); + + React.useEffect(() => { + if (isOpen && !loading && (!data || error)) { + showCustomAlert({ + title: 'Could not fetch stale status for assets', + body: 'An unknown error occurred.', + }); + onClose(); + } + }, [data, error, isOpen, loading, onClose]); + + const containerRef = React.useRef(null); + const virtualizer = useVirtualizer({ + count: staleOrMissing?.length ?? 0, + getScrollElement: () => containerRef.current, + estimateSize: () => 28, + }); + const totalHeight = virtualizer.getTotalSize(); + const items = virtualizer.getVirtualItems(); + + const [checked, setChecked] = React.useState>(new Set()); + React.useLayoutEffect(() => { + setChecked(new Set(staleOrMissing)); + }, [staleOrMissing]); + + const content = () => { + if (!isOpen) { + return null; + } + if (loading) { + return ( + + Fetching asset statuses + + ); + } + if (staleOrMissing?.length) { + return ( + <> + + { + setChecked((checked) => { + if (checked.size === staleOrMissing.length) { + return new Set(); + } else { + return new Set(staleOrMissing); + } + }); + }} + /> + + + + + {items.map(({index, key, size, start, measureElement}) => { + const item = staleOrMissing[index]!; + return ( + + + { + setChecked((checked) => { + const copy = new Set(checked); + if (copy.has(item)) { + copy.delete(item); + } else { + copy.add(item); + } + return copy; + }); + }} + /> + + + + + + + + + + + ); + })} + + + + ); + } + return ( + + +
All assets are up to date
+ + ); + }; + return ( + <> + + {content()} + + {loading ? ( + + ) : staleOrMissing?.length ? ( + + ) : ( + + )} + + + + ); + }, +); + +const ASSET_STALE_STATUS_QUERY = gql` + query AssetStaleStatusQuery($assetKeys: [AssetKeyInput!]!) { + assetNodes(assetKeys: $assetKeys) { + id + assetKey { + path + } + staleStatus + } + } +`; + +const TEMPLATE_COLUMNS = '20px minmax(0, 1fr)'; + +const RowGrid = styled(Box)` + display: grid; + grid-template-columns: ${TEMPLATE_COLUMNS}; + gap: 8px; + height: 100%; + align-items: center; +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx index 7bb49d51dd354..d4da79d1b32da 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx @@ -19,9 +19,7 @@ import {IExecutionSession} from '../app/ExecutionSessionStorage'; import { displayNameForAssetKey, isHiddenAssetGroupJob, - LiveData, sortAssetKeys, - toGraphId, tokenForAssetKey, } from '../asset-graph/Utils'; import {useLaunchPadHooks} from '../launchpad/LaunchpadHooksContext'; @@ -35,9 +33,9 @@ import {RepoAddress} from '../workspace/types'; import {ASSET_NODE_CONFIG_FRAGMENT} from './AssetConfig'; import {MULTIPLE_DEFINITIONS_WARNING} from './AssetDefinedInMultipleReposNotice'; +import {CalculateChangedAndMissingDialog} from './CalculateChangedAndMissingDialog'; import {LaunchAssetChoosePartitionsDialog} from './LaunchAssetChoosePartitionsDialog'; import {partitionDefinitionsEqual} from './MultipartitioningSupport'; -import {isAssetMissing, isAssetStale} from './Stale'; import {asAssetKeyInput, getAssetCheckHandleInputs} from './asInput'; import {AssetKey} from './types'; import { @@ -115,7 +113,7 @@ export const ERROR_INVALID_ASSET_SELECTION = ` the same code location and share a partition space, or form a connected` + ` graph in which root assets share the same partitioning.`; -function optionsForButton(scope: AssetsInScope, liveDataForStale?: LiveData): LaunchOption[] { +function optionsForButton(scope: AssetsInScope): LaunchOption[] { // If you pass a set of selected assets, we always show just one option // to materialize that selection. if ('selected' in scope) { @@ -142,20 +140,6 @@ function optionsForButton(scope: AssetsInScope, liveDataForStale?: LiveData): La : `Materialize${isAnyPartitioned(executable) ? '…' : ''}`, }); - if (liveDataForStale) { - const missingOrStale = executable.filter( - (a) => - isAssetMissing(liveDataForStale[toGraphId(a.assetKey)]) || - isAssetStale(liveDataForStale[toGraphId(a.assetKey)]), - ); - - options.push({ - assetKeys: missingOrStale.map((a) => a.assetKey), - label: `Materialize changed and missing${countOrBlank(missingOrStale)}`, - icon: , - }); - } - return options; } @@ -173,7 +157,6 @@ export function executionDisabledMessageForAssets( export const LaunchAssetExecutionButton: React.FC<{ scope: AssetsInScope; - liveDataForStale?: LiveData; // For "stale" dropdown options intent?: 'primary' | 'none'; preferredJobName?: string; additionalDropdownOptions?: { @@ -181,19 +164,16 @@ export const LaunchAssetExecutionButton: React.FC<{ icon?: JSX.Element; onClick: () => void; }[]; -}> = ({ - scope, - liveDataForStale, - preferredJobName, - additionalDropdownOptions, - intent = 'primary', -}) => { +}> = ({scope, preferredJobName, additionalDropdownOptions, intent = 'primary'}) => { const {onClick, loading, launchpadElement} = useMaterializationAction(preferredJobName); const [isOpen, setIsOpen] = React.useState(false); const {MaterializeButton} = useLaunchPadHooks(); - const options = optionsForButton(scope, liveDataForStale); + const [showCalculatingChangedAndMissingDialog, setShowCalculatingChangedAndMissingDialog] = + React.useState(false); + + const options = optionsForButton(scope); const firstOption = options[0]!; if (!firstOption) { return ; @@ -219,6 +199,16 @@ export const LaunchAssetExecutionButton: React.FC<{ return ( <> + { + setShowCalculatingChangedAndMissingDialog(false); + }} + assets={inScope} + onMaterializeAssets={(assets: AssetKey[], e: React.MouseEvent) => { + onClick(assets, e); + }} + /> onClick(option.assetKeys, e)} /> ))} + {inScope.length && 'all' in scope ? ( + setShowCalculatingChangedAndMissingDialog(true)} + /> + ) : null} } diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/Stale.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/Stale.tsx index 4b5191e2563ad..875404e380215 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/Stale.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/Stale.tsx @@ -22,10 +22,10 @@ import {assetDetailsPathForKey} from './assetDetailsPathForKey'; type StaleDataForNode = Pick; -export const isAssetMissing = (liveData?: StaleDataForNode) => +export const isAssetMissing = (liveData?: Pick) => liveData && liveData.staleStatus === StaleStatus.MISSING; -export const isAssetStale = (liveData?: StaleDataForNode) => +export const isAssetStale = (liveData?: Pick) => liveData && liveData.staleStatus === StaleStatus.STALE; const LABELS = { diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types/CalculateChangedAndMissingDialog.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/types/CalculateChangedAndMissingDialog.types.ts new file mode 100644 index 0000000000000..863281aa236a0 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types/CalculateChangedAndMissingDialog.types.ts @@ -0,0 +1,17 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../graphql/types'; + +export type AssetStaleStatusQueryVariables = Types.Exact<{ + assetKeys: Array | Types.AssetKeyInput; +}>; + +export type AssetStaleStatusQuery = { + __typename: 'Query'; + assetNodes: Array<{ + __typename: 'AssetNode'; + id: string; + staleStatus: Types.StaleStatus | null; + assetKey: {__typename: 'AssetKey'; path: Array}; + }>; +};