diff --git a/catalog/CHANGELOG.md b/catalog/CHANGELOG.md index 5bb0bf9fd17..8758f53dd0a 100644 --- a/catalog/CHANGELOG.md +++ b/catalog/CHANGELOG.md @@ -17,6 +17,7 @@ where verb is one of ## Changes +- [Changed] Athena: improve loading state and errors visuals; fix minor bugs; alphabetize and persist selection in workgroups, catalog names and databases ([#4208](https://github.com/quiltdata/quilt/pull/4208)) - [Changed] Show stack release version in footer ([#4200](https://github.com/quiltdata/quilt/pull/4200)) - [Added] Selective package downloading ([#4173](https://github.com/quiltdata/quilt/pull/4173)) - [Added] Qurator Omni: initial public release ([#4032](https://github.com/quiltdata/quilt/pull/4032), [#4181](https://github.com/quiltdata/quilt/pull/4181)) diff --git a/catalog/app/containers/Bucket/Queries/Athena/Athena.tsx b/catalog/app/containers/Bucket/Queries/Athena/Athena.tsx index b7c20d6f26c..9093a8159e2 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/Athena.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/Athena.tsx @@ -1,144 +1,156 @@ import cx from 'classnames' -import invariant from 'invariant' import * as R from 'ramda' import * as React from 'react' import * as RRDom from 'react-router-dom' import * as M from '@material-ui/core' import Code from 'components/Code' +import Placeholder from 'components/Placeholder' import Skeleton from 'components/Skeleton' +import * as BucketPreferences from 'utils/BucketPreferences' import * as NamedRoutes from 'utils/NamedRoutes' import QuerySelect from '../QuerySelect' -import * as requests from '../requests' -import { Alert, Section, makeAsyncDataErrorHandler } from './Components' -import CreatePackage from './CreatePackage' +import { Alert, Section } from './Components' import * as QueryEditor from './QueryEditor' -import Results from './Results' import History from './History' +import Results from './Results' import Workgroups from './Workgroups' +import * as Model from './model' +import { doQueryResultsContainManifestEntries } from './model/createPackage' -interface QuerySelectSkeletonProps { - className?: string -} +const CreatePackage = React.lazy(() => import('./CreatePackage')) -function QuerySelectSkeleton({ className }: QuerySelectSkeletonProps) { +function SeeDocsForCreatingPackage() { return ( -
- - -
+ + + + help_outline + + + ) } -const useAthenaQueriesStyles = M.makeStyles((t) => ({ - form: { - margin: t.spacing(3, 0, 0), +const useRelieveMessageStyles = M.makeStyles((t) => ({ + root: { + padding: t.spacing(2), + }, + text: { + animation: '$show 0.3s ease-out', + }, + '@keyframes show': { + from: { + opacity: 0, + }, + to: { + opacity: 1, + }, }, })) -interface QueryConstructorProps { - bucket: string - className?: string - initialValue?: requests.athena.QueryExecution - workgroup: requests.athena.Workgroup +const RELIEVE_INITIAL_TIMEOUT = 1000 + +interface RelieveMessageProps { + className: string + messages: string[] } -function QueryConstructor({ - bucket, - className, - initialValue, - workgroup, -}: QueryConstructorProps) { - const [query, setQuery] = React.useState(null) - const [prev, setPrev] = React.useState(null) - const data = requests.athena.useQueries(workgroup, prev) - const classes = useAthenaQueriesStyles() - const [value, setValue] = React.useState( - initialValue || null, +function RelieveMessage({ className, messages }: RelieveMessageProps) { + const classes = useRelieveMessageStyles() + const [relieve, setRelieve] = React.useState('') + const timersData = React.useMemo( + () => + messages.map((message, index) => ({ + timeout: RELIEVE_INITIAL_TIMEOUT * (index + 1) ** 2, + message, + })), + [messages], ) - const handleNamedQueryChange = React.useCallback( - (q: requests.athena.AthenaQuery | null) => { - setQuery(q) - setValue((x) => ({ - ...x, - query: q?.body, - })) - }, - [], + React.useEffect(() => { + const timers = timersData.map(({ timeout, message }) => + setTimeout(() => setRelieve(message), timeout), + ) + return () => { + timers.forEach((timer) => clearTimeout(timer)) + } + }, [timersData]) + if (!relieve) return null + return ( + + + {relieve} + + ) +} - const handleChange = React.useCallback((x: requests.athena.QueryExecution) => { - setValue(x) - setQuery(null) - }, []) +interface QuerySelectSkeletonProps { + className?: string +} +function QuerySelectSkeleton({ className }: QuerySelectSkeletonProps) { return (
- {data.case({ - Ok: (queries) => ( -
- {!!queries.list.length && ( - - onChange={handleNamedQueryChange} - onLoadMore={queries.next ? () => setPrev(queries) : undefined} - queries={queries.list} - value={query} - /> - )} -
- ), - Err: makeAsyncDataErrorHandler('Select query'), - _: () => , - })} - + +
) } -function QueryConstructorSkeleton() { - const classes = useStyles() +interface QueryConstructorProps { + className?: string +} + +function QueryConstructor({ className }: QueryConstructorProps) { + const { query, queries, queryRun } = Model.use() + + if (Model.isError(queries.data)) { + return + } + + if (!Model.hasData(queries.data) || !Model.isReady(query.value)) { + return + } + + if (!queries.data.list.length && !Model.isError(query.value)) { + return No saved queries. + } + return ( <> - - + + label="Select a query" + className={className} + disabled={Model.isLoading(queryRun)} + onChange={query.setValue} + onLoadMore={queries.data.next ? queries.loadMore : undefined} + queries={queries.data.list} + value={Model.isError(query.value) ? null : query.value} + /> + {Model.isError(query.value) && ( + {query.value.message} + )} ) } -interface HistoryContainerProps { - bucket: string - className: string - workgroup: requests.athena.Workgroup -} - -function HistoryContainer({ bucket, className, workgroup }: HistoryContainerProps) { - const [prev, setPrev] = React.useState( - null, - ) - const data = requests.athena.useQueryExecutions(workgroup, prev) +function HistoryContainer() { + const { bucket, executions } = Model.use() + if (Model.isError(executions.data)) { + return + } + if (!Model.hasData(executions.data)) { + return + } return ( -
- {data.case({ - Ok: (executions) => ( - setPrev(executions) : undefined} - workgroup={workgroup} - /> - ), - Err: makeAsyncDataErrorHandler('Executions Data'), - _: () => , - })} -
+ ) } @@ -146,89 +158,90 @@ const useResultsContainerStyles = M.makeStyles((t) => ({ breadcrumbs: { margin: t.spacing(0, 0, 1), }, + relieve: { + left: '50%', + position: 'absolute', + top: t.spacing(7), + transform: 'translateX(-50%)', + }, + table: { + position: 'relative', + }, })) interface ResultsContainerSkeletonProps { bucket: string className: string - queryExecutionId: string - workgroup: requests.athena.Workgroup } -function ResultsContainerSkeleton({ - bucket, - className, - queryExecutionId, - workgroup, -}: ResultsContainerSkeletonProps) { +const relieveMessages = [ + 'Still loading…', + 'This is taking a moment. Thanks for your patience!', + 'Looks like a heavy task! We’re still working on it.', + 'Hang in there, we haven’t forgotten about you! Your request is still being processed.', +] + +function ResultsContainerSkeleton({ bucket, className }: ResultsContainerSkeletonProps) { const classes = useResultsContainerStyles() return (
- + - +
+ + +
) } interface ResultsContainerProps { - bucket: string className: string - queryExecutionId: string - queryResults: requests.athena.QueryResultsResponse - workgroup: requests.athena.Workgroup - onLoadMore?: () => void } -function ResultsContainer({ - bucket, - className, - queryExecutionId, - queryResults, - onLoadMore, - workgroup, -}: ResultsContainerProps) { +function ResultsContainer({ className }: ResultsContainerProps) { const classes = useResultsContainerStyles() + const { bucket, execution, results } = Model.use() + + if (Model.isError(execution)) { + return ( +
+ + +
+ ) + } + + if (Model.isError(results.data)) { + return ( +
+ + +
+ ) + } + + if (!Model.isReady(execution) || !Model.isReady(results.data)) { + return + } + return (
- - {!!queryResults.rows.length && ( - + + {doQueryResultsContainManifestEntries(results.data) ? ( + }> + + + ) : ( + )} - {/* eslint-disable-next-line no-nested-ternary */} - {queryResults.rows.length ? ( - - ) : // eslint-disable-next-line no-nested-ternary - queryResults.queryExecution.error ? ( - - ) : queryResults.queryExecution ? ( - - ) : ( - - )} +
) } @@ -248,19 +261,6 @@ function TableSkeleton({ size }: TableSkeletonProps) { ) } -interface QueryResults { - data: requests.AsyncData - loadMore: (prev: requests.athena.QueryResultsResponse) => void -} - -function useQueryResults(queryExecutionId?: string): QueryResults { - const [prev, setPrev] = React.useState( - null, - ) - const data = requests.athena.useQueryResults(queryExecutionId || null, prev) - return React.useMemo(() => ({ data, loadMore: setPrev }), [data]) -} - const useOverrideStyles = M.makeStyles({ li: { '&::before': { @@ -278,7 +278,7 @@ const useResultsBreadcrumbsStyles = M.makeStyles({ display: 'flex', }, actions: { - marginLeft: 'auto', + margin: '-3px 0 -3px auto', }, breadcrumb: { display: 'flex', @@ -290,19 +290,12 @@ const useResultsBreadcrumbsStyles = M.makeStyles({ interface ResultsBreadcrumbsProps { bucket: string - children: React.ReactNode + children?: React.ReactNode className?: string - queryExecutionId?: string - workgroup: requests.athena.Workgroup } -function ResultsBreadcrumbs({ - bucket, - children, - className, - queryExecutionId, - workgroup, -}: ResultsBreadcrumbsProps) { +function ResultsBreadcrumbs({ bucket, children, className }: ResultsBreadcrumbsProps) { + const { workgroup, queryExecutionId } = Model.use() const classes = useResultsBreadcrumbsStyles() const overrideClasses = useOverrideStyles() const { urls } = NamedRoutes.use() @@ -311,7 +304,7 @@ function ResultsBreadcrumbs({ Query Executions @@ -320,7 +313,7 @@ function ResultsBreadcrumbs({ -
{children}
+ {children &&
{children}
} ) } @@ -335,97 +328,13 @@ const useStyles = M.makeStyles((t) => ({ section: { margin: t.spacing(3, 0, 0), }, + form: { + margin: t.spacing(3, 0, 0), + }, })) -interface AthenaMainProps { - bucket: string - workgroup: string -} - -function AthenaMain({ bucket, workgroup }: AthenaMainProps) { - const classes = useStyles() - const data = requests.athena.useDefaultQueryExecution() - return ( -
- {data.case({ - Ok: (queryExecution) => ( - - ), - Err: makeAsyncDataErrorHandler('Default catalog and database'), - _: () => , - })} - -
- ) -} - -interface AthenaExecutionProps { - bucket: string - queryExecutionId: string - workgroup: string -} - -function AthenaExecution({ bucket, workgroup, queryExecutionId }: AthenaExecutionProps) { - const classes = useStyles() - const results = useQueryResults(queryExecutionId) - return ( -
- {results.data.case({ - Ok: (value) => ( - - ), - _: () => , - })} - - {results.data.case({ - Ok: (queryResults) => ( - results.loadMore(queryResults) : undefined - } - workgroup={workgroup} - /> - ), - _: () => ( - - ), - Err: makeAsyncDataErrorHandler('Query Results Data'), - })} -
- ) -} - -export default function AthenaContainer() { - const { bucket, queryExecutionId, workgroup } = RRDom.useParams<{ - bucket: string - queryExecutionId?: string - workgroup?: string - }>() - invariant(!!bucket, '`bucket` must be defined') +function AthenaContainer() { + const { bucket, queryExecutionId, workgroup } = Model.use() const classes = useStyles() return ( @@ -434,18 +343,38 @@ export default function AthenaContainer() { Athena SQL - - - {workgroup && - (queryExecutionId ? ( - - ) : ( - - ))} + + + {Model.hasData(workgroup.data) && ( +
+
+ + +
+ {queryExecutionId ? ( + + ) : ( +
+ +
+ )} +
+ )} ) } + +export default function Wrapper() { + const prefs = BucketPreferences.use() + return BucketPreferences.Result.match( + { + Ok: ({ ui }) => ( + + + + ), + _: () => , + }, + prefs, + ) +} diff --git a/catalog/app/containers/Bucket/Queries/Athena/Components.tsx b/catalog/app/containers/Bucket/Queries/Athena/Components.tsx index 4521962c352..e4114ff055c 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/Components.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/Components.tsx @@ -3,11 +3,10 @@ import * as React from 'react' import * as M from '@material-ui/core' import * as Lab from '@material-ui/lab' -import * as Sentry from 'utils/Sentry' - const useSectionStyles = M.makeStyles((t) => ({ header: { - margin: t.spacing(0, 0, 1), + ...t.typography.body1, + margin: t.spacing(2, 0, 1), }, })) @@ -20,36 +19,28 @@ interface SectionProps { export function Section({ className, empty, title, children }: SectionProps) { const classes = useSectionStyles() - if (!children && empty) - return {empty} + if (!children && empty) { + return
{empty}
+ } return (
- {title} +
{title}
{children}
) } interface AlertProps { + className?: string error: Error title: string } -export function Alert({ error, title }: AlertProps) { - const sentry = Sentry.use() - - React.useEffect(() => { - sentry('captureException', error) - }, [error, sentry]) - +export function Alert({ className, error, title }: AlertProps) { return ( - + {title} {error.message} ) } - -export function makeAsyncDataErrorHandler(title: string) { - return (error: Error) => -} diff --git a/catalog/app/containers/Bucket/Queries/Athena/CreatePackage.tsx b/catalog/app/containers/Bucket/Queries/Athena/CreatePackage.tsx index 43516279e67..5d0229e22c0 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/CreatePackage.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/CreatePackage.tsx @@ -4,114 +4,16 @@ import * as M from '@material-ui/core' import * as Dialog from 'components/Dialog' import * as AddToPackage from 'containers/AddToPackage' import { usePackageCreationDialog } from 'containers/Bucket/PackageDialog/PackageCreationForm' -import type * as Model from 'model' -import * as s3paths from 'utils/s3paths' -import * as requests from '../requests' +import type * as requests from './model/requests' +import { + ParsedRows, + doQueryResultsContainManifestEntries, + parseQueryResults, +} from './model/createPackage' import Results from './Results' -type ManifestKey = 'hash' | 'logical_key' | 'meta' | 'physical_keys' | 'size' -type ManifestEntryStringified = Record - -function SeeDocsForCreatingPackage() { - return ( - - - - help_outline - - - - ) -} - -function doQueryResultsContainManifestEntries( - queryResults: requests.athena.QueryResultsResponse, -): queryResults is requests.athena.QueryManifestsResponse { - const columnNames = queryResults.columns.map(({ name }) => name) - return ( - columnNames.includes('size') && - columnNames.includes('physical_keys') && - columnNames.includes('logical_key') - ) -} - -// TODO: this name doesn't make sense without `parseManifestEntryStringified` -// merge it into one -function rowToManifestEntryStringified( - row: string[], - columns: requests.athena.QueryResultsColumns, -): ManifestEntryStringified { - return row.reduce((acc, value, index) => { - if (!columns[index].name) return acc - return { - ...acc, - [columns[index].name]: value, - } - }, {} as ManifestEntryStringified) -} - -function parseManifestEntryStringified(entry: ManifestEntryStringified): { - [key: string]: Model.S3File -} | null { - if (!entry.logical_key) return null - if (!entry.physical_keys) return null - try { - const handle = s3paths.parseS3Url( - entry.physical_keys.replace(/^\[/, '').replace(/\]$/, ''), - ) - const sizeParsed = Number(entry.size) - const size = Number.isNaN(sizeParsed) ? 0 : sizeParsed - return { - [entry.logical_key]: { - ...handle, - size, - }, - } - } catch (e) { - // eslint-disable-next-line no-console - console.error(e) - return null - } -} - -interface ParsedRows { - valid: Record - invalid: requests.athena.QueryResultsRows -} - -function parseQueryResults( - queryResults: requests.athena.QueryManifestsResponse, -): ParsedRows { - // TODO: use one reduce-loop - // merge `rowToManifestEntryStringified` and `parseManifestEntryStringified` into one function - const manifestEntries: ManifestEntryStringified[] = queryResults.rows.reduce( - (memo, row) => memo.concat(rowToManifestEntryStringified(row, queryResults.columns)), - [] as ManifestEntryStringified[], - ) - return manifestEntries.reduce( - (memo, entry, index) => { - const parsed = parseManifestEntryStringified(entry) - return parsed - ? // if entry is ok then add it to valid map, and invalid is pristine - { - valid: { - ...memo.valid, - ...parsed, - }, - invalid: memo.invalid, - } - : // if no entry then add original data to list of invalid, and valid is pristine - { - valid: memo.valid, - invalid: [...memo.invalid, queryResults.rows[index]], - } - }, - { valid: {}, invalid: [] } as ParsedRows, - ) -} - const useStyles = M.makeStyles((t) => ({ results: { 'div&': { @@ -123,7 +25,7 @@ const useStyles = M.makeStyles((t) => ({ interface CreatePackageProps { bucket: string - queryResults: requests.athena.QueryResultsResponse + queryResults: requests.QueryResults } export default function CreatePackage({ bucket, queryResults }: CreatePackageProps) { @@ -150,7 +52,6 @@ export default function CreatePackage({ bucket, queryResults }: CreatePackagePro const onPackage = React.useCallback(() => { if (!doQueryResultsContainManifestEntries(queryResults)) return - // TODO: make it lazy, and disable button const parsed = parseQueryResults(queryResults) setEntries(parsed) if (parsed.invalid.length) { @@ -161,10 +62,6 @@ export default function CreatePackage({ bucket, queryResults }: CreatePackagePro } }, [addToPackage, confirm, createDialog, queryResults]) - if (!doQueryResultsContainManifestEntries(queryResults)) { - return - } - return ( <> {createDialog.render({ diff --git a/catalog/app/containers/Bucket/Queries/Athena/Database.spec.tsx b/catalog/app/containers/Bucket/Queries/Athena/Database.spec.tsx new file mode 100644 index 00000000000..582b1a6d433 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/Database.spec.tsx @@ -0,0 +1,144 @@ +import * as React from 'react' +import renderer from 'react-test-renderer' + +import WithGlobalDialogs from 'utils/GlobalDialogs' + +import Database from './Database' + +import * as Model from './model' + +jest.mock( + 'constants/config', + jest.fn(() => ({})), +) + +const noop = () => {} + +const emptyState: Model.State = { + bucket: 'any', + + catalogName: { value: undefined, setValue: noop }, + catalogNames: { data: undefined, loadMore: noop }, + database: { value: undefined, setValue: noop }, + databases: { data: undefined, loadMore: noop }, + execution: undefined, + executions: { data: undefined, loadMore: noop }, + queries: { data: undefined, loadMore: noop }, + query: { value: undefined, setValue: noop }, + queryBody: { value: undefined, setValue: noop }, + results: { data: undefined, loadMore: noop }, + workgroups: { data: undefined, loadMore: noop }, + workgroup: { data: undefined, loadMore: noop }, + + submit: () => Promise.resolve({ id: 'bar' }), + queryRun: undefined, +} + +interface ProviderProps { + children: React.ReactNode + value: Model.State +} + +function Provider({ children, value }: ProviderProps) { + return {children} +} + +describe('containers/Bucket/Queries/Athena/Database', () => { + beforeAll(() => {}) + + afterAll(() => {}) + + it('should render skeletons', () => { + const tree = renderer.create( + + + , + ) + expect(tree).toMatchSnapshot() + }) + + it('should render selected values', () => { + const tree = renderer.create( + + + , + ) + expect(tree).toMatchSnapshot() + }) + + it('should show no value (zero-width space) if selected no value', () => { + const tree = renderer.create( + + + , + ) + expect(tree).toMatchSnapshot() + }) + + it('should disable selection if no spare values', () => { + const tree = renderer.create( + + + , + ) + expect(tree).toMatchSnapshot() + }) + + it('should show error when values failed', () => { + const tree = renderer.create( + + + + + , + ) + expect(tree).toMatchSnapshot() + }) + + it('should show error when data failed', () => { + const tree = renderer.create( + + + + + , + ) + expect(tree).toMatchSnapshot() + }) +}) diff --git a/catalog/app/containers/Bucket/Queries/Athena/Database.tsx b/catalog/app/containers/Bucket/Queries/Athena/Database.tsx index dbe46d38b72..7f73a8e36f6 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/Database.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/Database.tsx @@ -6,7 +6,17 @@ import * as Lab from '@material-ui/lab' import Skeleton from 'components/Skeleton' import * as Dialogs from 'utils/GlobalDialogs' -import * as requests from '../requests' +import * as Model from './model' +import * as storage from './model/storage' + +const useSelectErrorStyles = M.makeStyles((t) => ({ + button: { + whiteSpace: 'nowrap', + }, + dialog: { + padding: t.spacing(2), + }, +})) interface SelectErrorProps { className?: string @@ -14,18 +24,24 @@ interface SelectErrorProps { } function SelectError({ className, error }: SelectErrorProps) { + const classes = useSelectErrorStyles() const openDialog = Dialogs.use() const handleClick = React.useCallback(() => { openDialog(() => ( - +
{error.message} - +
)) - }, [error.message, openDialog]) + }, [classes.dialog, error.message, openDialog]) return ( + Show more } @@ -39,6 +55,8 @@ function SelectError({ className, error }: SelectErrorProps) { const LOAD_MORE = '__load-more__' +const EMPTY = '__empty__' + interface Response { list: string[] next?: string @@ -84,13 +102,22 @@ function Select({ return ( {label} - + {data.list.map((item) => ( {item} ))} {data.next && Load more} + {!data.list.length && ( + + {value || 'Empty list'} + + )} ) @@ -98,54 +125,78 @@ function Select({ interface SelectCatalogNameProps { className?: string - value: requests.athena.CatalogName | null - onChange: (catalogName: requests.athena.CatalogName) => void } -function SelectCatalogName({ className, value, onChange }: SelectCatalogNameProps) { - const [prev, setPrev] = React.useState( - null, +function SelectCatalogName({ className }: SelectCatalogNameProps) { + const { catalogName, catalogNames, queryRun } = Model.use() + + const handleChange = React.useCallback( + (value) => { + storage.setCatalog(value) + storage.clearDatabase() + catalogName.setValue(value) + }, + [catalogName], + ) + + if (Model.isError(catalogNames.data)) { + return + } + if (Model.isError(catalogName.value)) { + return + } + if (!Model.hasValue(catalogName.value) || !Model.hasData(catalogNames.data)) { + return + } + + return ( + - ), - Err: (error) => , - _: () => , - }) } -interface SelectDatabaseProps - extends Omit { - catalogName: requests.athena.CatalogName | null - onChange: (database: requests.athena.Database) => void - value: requests.athena.Database | null +interface SelectDatabaseProps { + className: string } -function SelectDatabase({ catalogName, onChange, ...rest }: SelectDatabaseProps) { - const [prev, setPrev] = React.useState(null) - const data = requests.athena.useDatabases(catalogName, prev) - return data.case({ - Ok: (response) => ( - + ) } const useStyles = M.makeStyles((t) => ({ @@ -155,8 +206,8 @@ const useStyles = M.makeStyles((t) => ({ }, field: { cursor: 'pointer', + flexBasis: '50%', marginRight: t.spacing(2), - width: '50%', '& input': { cursor: 'pointer', }, @@ -171,42 +222,14 @@ const useStyles = M.makeStyles((t) => ({ interface DatabaseProps { className?: string - value: requests.athena.ExecutionContext | null - onChange: (value: requests.athena.ExecutionContext | null) => void } -export default function Database({ className, value, onChange }: DatabaseProps) { +export default function Database({ className }: DatabaseProps) { const classes = useStyles() - const [catalogName, setCatalogName] = - React.useState(value?.catalogName || null) - const handleCatalogName = React.useCallback( - (name) => { - setCatalogName(name) - onChange(null) - }, - [onChange], - ) - const handleDatabase = React.useCallback( - (database) => { - if (!catalogName) return - onChange({ catalogName, database }) - }, - [catalogName, onChange], - ) return (
- - + +
) } diff --git a/catalog/app/containers/Bucket/Queries/Athena/History.tsx b/catalog/app/containers/Bucket/Queries/Athena/History.tsx index 2653c70a7be..23de3ea9811 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/History.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/History.tsx @@ -2,16 +2,15 @@ import cx from 'classnames' import * as dateFns from 'date-fns' import * as R from 'ramda' import * as React from 'react' +import * as RRDom from 'react-router-dom' import * as M from '@material-ui/core' import * as Lab from '@material-ui/lab' import * as Notifications from 'containers/Notifications' import * as NamedRoutes from 'utils/NamedRoutes' -import Link from 'utils/StyledLink' import copyToClipboard from 'utils/clipboard' -import { trimCenter } from 'utils/string' -import * as requests from '../requests' +import * as Model from './model' const useToggleButtonStyles = M.makeStyles({ root: { @@ -58,133 +57,154 @@ function Date({ date }: DateProps) { return {formatted} } -interface QueryDateCompletedProps { - bucket: string - queryExecution: requests.athena.QueryExecution - workgroup: requests.athena.Workgroup -} - -function QueryDateCompleted({ - bucket, - queryExecution, - workgroup, -}: QueryDateCompletedProps) { - const { urls } = NamedRoutes.use() - if (queryExecution.status !== 'SUCCEEDED') { - return - } - return ( - - - - ) -} - -interface CopyButtonProps { - queryExecution: requests.athena.QueryExecution -} - -function CopyButton({ queryExecution }: CopyButtonProps) { - const { push } = Notifications.use() - const handleCopy = React.useCallback(() => { - if (queryExecution.query) { - copyToClipboard(queryExecution.query) - push('Query has been copied to clipboard') - } - }, [push, queryExecution.query]) - return ( - - content_copy - - ) -} - const useFullQueryRowStyles = M.makeStyles((t) => ({ - cell: { - paddingBottom: 0, - paddingTop: 0, - }, - collapsed: { - borderBottom: 0, + root: { + borderBottom: `1px solid ${t.palette.divider}`, + padding: t.spacing(2, 7.5), }, query: { maxHeight: t.spacing(30), maxWidth: '100%', overflow: 'auto', + margin: t.spacing(0, 0, 2), + whiteSpace: 'pre-wrap', + }, + button: { + '& + &': { + marginLeft: t.spacing(1), + }, }, })) interface FullQueryRowProps { expanded: boolean - queryExecution: requests.athena.QueryExecution + query: string } -function FullQueryRow({ expanded, queryExecution }: FullQueryRowProps) { +function FullQueryRow({ expanded, query }: FullQueryRowProps) { + const { push } = Notifications.use() + const { queryBody } = Model.use() const classes = useFullQueryRowStyles() + const handleInsert = React.useCallback(() => { + queryBody.setValue(query) + push('Query has been pasted into editor') + }, [push, queryBody, query]) + const handleCopy = React.useCallback(() => { + copyToClipboard(query) + push('Query has been copied to clipboard') + }, [push, query]) return ( - - - {!!expanded && } - - - -
{queryExecution.query}
-
-
-
+ +
+
{query}
+ content_copy} + variant="outlined" + > + Copy + + replay} + variant="outlined" + > + Paste into query editor + +
+
) } -interface ExecutionProps { - bucket: string - queryExecution: requests.athena.QueryExecution - workgroup: requests.athena.Workgroup +const useRowStyles = M.makeStyles((t) => ({ + root: { + alignItems: 'center', + display: 'grid', + gridColumnGap: t.spacing(2), + gridTemplateColumns: '30px auto 160px 160px 160px', + padding: t.spacing(0, 2), + lineHeight: `${t.spacing(4)}px`, + borderBottom: `1px solid ${t.palette.divider}`, + whiteSpace: 'nowrap', + }, +})) + +interface RowProps { + className: string + children: React.ReactNode } -function Execution({ bucket, queryExecution, workgroup }: ExecutionProps) { - const [expanded, setExpanded] = React.useState(false) - const onToggle = React.useCallback(() => setExpanded(!expanded), [expanded]) +function Row({ className, children }: RowProps) { + const classes = useRowStyles() + return
{children}
+} + +interface LinkCellProps { + children: React.ReactNode + className: string + to?: string +} - if (queryExecution.error) +function LinkCell({ children, className, to }: LinkCellProps) { + if (to) { return ( - - - {queryExecution.error.message} - - + + {children} + ) + } + return {children} +} + +const useExecutionStyles = M.makeStyles((t) => ({ + hover: { + '&:has($link:hover)': { + background: t.palette.action.hover, + }, + }, + failed: { + color: t.palette.text.disabled, + }, + link: {}, + query: { + overflow: 'hidden', + textOverflow: 'ellipsis', + }, +})) + +interface ExecutionProps { + to?: string + queryExecution: Model.QueryExecution +} + +function Execution({ to, queryExecution }: ExecutionProps) { + const classes = useExecutionStyles() + const [expanded, setExpanded] = React.useState(false) + const onToggle = React.useCallback(() => setExpanded(!expanded), [expanded]) return ( <> - - - - - {trimCenter(queryExecution.query || '', 50)} - + + + + {queryExecution.query} + + {queryExecution.status || 'UNKNOWN'} - - + + - - - - - + + + + + {queryExecution.query && ( - + )} ) @@ -199,48 +219,42 @@ function Empty() { ) } +function isFailedExecution( + x: Model.QueryExecutionsItem, +): x is Model.QueryExecutionFailed { + return !!(x as Model.QueryExecutionFailed).error +} + const useStyles = M.makeStyles((t) => ({ - queryCell: { - width: '40%', - }, - actionCell: { - width: '24px', - }, header: { - margin: t.spacing(0, 0, 1), + lineHeight: `${t.spacing(4.5)}px`, + fontWeight: 500, }, footer: { + alignItems: 'center', display: 'flex', - padding: t.spacing(1), + padding: t.spacing(1, 2), }, more: { marginLeft: 'auto', }, - table: { - tableLayout: 'fixed', - }, })) interface HistoryProps { bucket: string - executions: requests.athena.QueryExecution[] + executions: Model.QueryExecutionsItem[] onLoadMore?: () => void - workgroup: requests.athena.Workgroup } -export default function History({ - bucket, - executions, - onLoadMore, - workgroup, -}: HistoryProps) { +export default function History({ bucket, executions, onLoadMore }: HistoryProps) { + const { urls } = NamedRoutes.use() const classes = useStyles() const pageSize = 10 const [page, setPage] = React.useState(1) const handlePagination = React.useCallback( - (event, value) => { + (_event, value) => { setPage(value) }, [setPage], @@ -249,8 +263,8 @@ export default function History({ const rowsSorted = React.useMemo( () => R.sort( - (a: requests.athena.QueryExecution, b: requests.athena.QueryExecution) => - b?.completed && a?.completed + (a: Model.QueryExecutionsItem, b: Model.QueryExecutionsItem) => + !isFailedExecution(a) && !isFailedExecution(b) && b?.completed && a?.completed ? b.completed.valueOf() - a.completed.valueOf() : -1, executions, @@ -260,54 +274,55 @@ export default function History({ const rowsPaginated = rowsSorted.slice(pageSize * (page - 1), pageSize * page) const hasPagination = rowsSorted.length > rowsPaginated.length + const { workgroup } = Model.use() + if (!Model.hasValue(workgroup)) return null + return ( - - - - - - Query - Status - Date created - Date completed - - - - {rowsPaginated.map((queryExecution) => ( + <> + + +
+ Query + Status + Date created + Date completed + + {rowsPaginated.map((queryExecution) => + isFailedExecution(queryExecution) ? ( + + {queryExecution.error.message} + + ) : ( - ))} - {!executions.length && ( - - - - - - )} - - - - {(hasPagination || !!onLoadMore) && ( -
- {hasPagination && ( - - )} - {onLoadMore && ( - - Load more - - )} -
- )} - + ), + )} + {!executions.length && } + {(hasPagination || !!onLoadMore) && ( +
+ {hasPagination && ( + + )} + {onLoadMore && ( + + Load more + + )} +
+ )} + + ) } diff --git a/catalog/app/containers/Bucket/Queries/Athena/QueryEditor.tsx b/catalog/app/containers/Bucket/Queries/Athena/QueryEditor.tsx index 584f53ff45f..f3a6b04746a 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/QueryEditor.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/QueryEditor.tsx @@ -1,21 +1,18 @@ import * as React from 'react' import AceEditor from 'react-ace' -import * as RRDom from 'react-router-dom' import * as M from '@material-ui/core' import * as Lab from '@material-ui/lab' import 'ace-builds/src-noconflict/mode-sql' import 'ace-builds/src-noconflict/theme-eclipse' -import { useConfirm } from 'components/Dialog' +import Lock from 'components/Lock' import Skeleton from 'components/Skeleton' -import * as Notifications from 'containers/Notifications' -import * as NamedRoutes from 'utils/NamedRoutes' +import * as Dialogs from 'utils/GlobalDialogs' import StyledLink from 'utils/StyledLink' -import * as requests from '../requests' - import Database from './Database' +import * as Model from './model' const ATHENA_REF_INDEX = 'https://aws.amazon.com/athena/' const ATHENA_REF_SQL = @@ -46,23 +43,31 @@ function HelperText() { const useStyles = M.makeStyles((t) => ({ editor: { padding: t.spacing(1), + position: 'relative', }, header: { - margin: t.spacing(0, 0, 1), + margin: t.spacing(2, 0, 1), }, })) -interface EditorFieldProps { - className?: string - onChange: (value: string) => void - query: string -} - -function EditorField({ className, query, onChange }: EditorFieldProps) { +function EditorField() { const classes = useStyles() + const { queryBody, queryRun } = Model.use() + + if (Model.isNone(queryBody.value)) { + return null + } + + if (Model.isError(queryBody.value)) { + return {queryBody.value.message} + } + + if (!Model.hasValue(queryBody.value)) { + return + } return ( -
+
Query body @@ -71,68 +76,19 @@ function EditorField({ className, query, onChange }: EditorFieldProps) { editorProps={{ $blockScrolling: true }} height="200px" mode="sql" - onChange={onChange} + onChange={queryBody.setValue} theme="eclipse" - value={query} + value={queryBody.value || ''} width="100%" /> + {Model.isLoading(queryRun) && }
) } -function useQueryRun( - bucket: string, - workgroup: requests.athena.Workgroup, - queryExecutionId?: string, -) { - const { urls } = NamedRoutes.use() - const history = RRDom.useHistory() - const [loading, setLoading] = React.useState(false) - const [error, setError] = React.useState() - const runQuery = requests.athena.useQueryRun(workgroup) - const { push: notify } = Notifications.use() - const goToExecution = React.useCallback( - (id: string) => history.push(urls.bucketAthenaExecution(bucket, workgroup, id)), - [bucket, history, urls, workgroup], - ) - const onSubmit = React.useCallback( - async (value: string, executionContext: requests.athena.ExecutionContext | null) => { - setLoading(true) - setError(undefined) - try { - const { id } = await runQuery(value, executionContext) - if (id === queryExecutionId) notify('Query execution results remain unchanged') - setLoading(false) - goToExecution(id) - } catch (e) { - setLoading(false) - if (e instanceof Error) { - setError(e) - } else { - throw e - } - } - }, - [goToExecution, notify, runQuery, queryExecutionId], - ) - return React.useMemo( - () => ({ - loading, - error, - onSubmit, - }), - [loading, error, onSubmit], - ) -} - const useFormSkeletonStyles = M.makeStyles((t) => ({ - button: { - height: t.spacing(4), - marginTop: t.spacing(2), - width: t.spacing(14), - }, canvas: { flexGrow: 1, height: t.spacing(27), @@ -157,7 +113,7 @@ const useFormSkeletonStyles = M.makeStyles((t) => ({ })) interface FormSkeletonProps { - className: string + className?: string } function FormSkeleton({ className }: FormSkeletonProps) { @@ -170,18 +126,43 @@ function FormSkeleton({ className }: FormSkeletonProps) {
-
) } +interface FormConfirmProps { + close: () => void + submit: () => void +} + +function FormConfirm({ close, submit }: FormConfirmProps) { + return ( + <> + + Database is not selected. Run the query without it? + + + Close + { + close() + submit() + }} + > + Confirm, run without + + + + ) +} + export { FormSkeleton as Skeleton } const useFormStyles = M.makeStyles((t) => ({ actions: { display: 'flex', justifyContent: 'space-between', - margin: t.spacing(2, 0), + margin: t.spacing(2, 0, 4), [t.breakpoints.up('sm')]: { alignItems: 'center', }, @@ -203,86 +184,38 @@ const useFormStyles = M.makeStyles((t) => ({ })) interface FormProps { - bucket: string - className?: string - onChange: (value: requests.athena.QueryExecution) => void - value: requests.athena.QueryExecution | null - workgroup: requests.athena.Workgroup + className: string } -export function Form({ bucket, className, onChange, value, workgroup }: FormProps) { +export function Form({ className }: FormProps) { const classes = useFormStyles() - const executionContext = React.useMemo( - () => - value?.catalog && value?.db - ? { - catalogName: value.catalog, - database: value.db, - } - : null, - [value], - ) - const confirm = useConfirm({ - onSubmit: (confirmed) => { - if (confirmed) { - if (!value?.query) { - throw new Error('Query is not set') - } - onSubmit(value!.query, executionContext) - } - }, - submitTitle: 'Proceed', - title: 'Execution context is not set', - }) - const { loading, error, onSubmit } = useQueryRun(bucket, workgroup, value?.id) - const handleSubmit = React.useCallback(() => { - if (!value?.query) return - if (!executionContext) { - return confirm.open() + const { submit, queryRun } = Model.use() + + const openDialog = Dialogs.use() + const handleSubmit = React.useCallback(async () => { + const output = await submit(false) + if (output === Model.NO_DATABASE) { + openDialog(({ close }) => submit(true)} />) } - onSubmit(value.query, executionContext) - }, [confirm, executionContext, onSubmit, value]) - const handleExecutionContext = React.useCallback( - (exeContext) => { - if (!exeContext) { - onChange({ ...value, catalog: undefined, db: undefined }) - return - } - const { catalogName, database } = exeContext - onChange({ ...value, catalog: catalogName, db: database }) - }, - [onChange, value], - ) + }, [openDialog, submit]) return (
- {confirm.render( - - Data catalog and database are not set. Run query without them? - , - )} - onChange({ ...value, query })} - query={value?.query || ''} - /> + - {error && ( + {Model.isError(queryRun) && ( - {error.message} + {queryRun.message} )}
- + Run query diff --git a/catalog/app/containers/Bucket/Queries/Athena/Results.tsx b/catalog/app/containers/Bucket/Queries/Athena/Results.tsx index f454f3ca198..e0ba16741df 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/Results.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/Results.tsx @@ -9,7 +9,7 @@ import log from 'utils/Logging' import * as NamedRoutes from 'utils/NamedRoutes' import * as s3paths from 'utils/s3paths' -import * as requests from '../requests' +import * as Model from './model' function Empty() { return ( @@ -63,9 +63,9 @@ const useResultsStyles = M.makeStyles((t) => ({ interface ResultsProps { className?: string - columns: requests.athena.QueryResultsColumns + columns: Model.QueryResultsColumns onLoadMore?: () => void - rows: requests.athena.QueryResultsRows + rows: Model.QueryResultsRows } export default function Results({ className, columns, onLoadMore, rows }: ResultsProps) { diff --git a/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx b/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx index 14209c44b33..b9713539063 100644 --- a/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx +++ b/catalog/app/containers/Bucket/Queries/Athena/Workgroups.tsx @@ -4,40 +4,31 @@ import * as M from '@material-ui/core' import * as Lab from '@material-ui/lab' import { docs } from 'constants/urls' -import * as NamedRoutes from 'utils/NamedRoutes' import Skeleton from 'components/Skeleton' +import * as NamedRoutes from 'utils/NamedRoutes' import StyledLink from 'utils/StyledLink' -import * as requests from '../requests' -import * as storage from '../requests/storage' - -import { Alert, Section } from './Components' - -const useStyles = M.makeStyles((t) => ({ - selectWrapper: { - width: '100%', - }, - select: { - padding: t.spacing(1), - }, -})) +import { Alert } from './Components' +import * as Model from './model' +import * as storage from './model/storage' const LOAD_MORE = 'load-more' interface WorkgroupSelectProps { bucket: string - onLoadMore: (workgroups: requests.athena.WorkgroupsResponse) => void - value: requests.athena.Workgroup | null - workgroups: requests.athena.WorkgroupsResponse + disabled?: boolean + onLoadMore: (workgroups: Model.List) => void + value: Model.Workgroup | null + workgroups: Model.List } function WorkgroupSelect({ bucket, + disabled, onLoadMore, value, workgroups, }: WorkgroupSelectProps) { - const classes = useStyles() const { urls } = NamedRoutes.use() const history = RRDom.useHistory() @@ -61,29 +52,27 @@ function WorkgroupSelect({ ) return ( - - - - {workgroups.list.map((name) => ( - - {name} - - ))} - {workgroups.next && ( - - - Load more - - - )} - - - + + Select workgroup + + {workgroups.list.map((name) => ( + + {name} + + ))} + {workgroups.next && ( + + + Load more + + + )} + + ) } @@ -116,54 +105,33 @@ function WorkgroupsEmpty({ error }: WorkgroupsEmptyProps) { ) } -interface RedirectToDefaultWorkgroupProps { - bucket: string - workgroups: requests.athena.WorkgroupsResponse -} - -function RedirectToDefaultWorkgroup({ - bucket, - workgroups, -}: RedirectToDefaultWorkgroupProps) { - const { urls } = NamedRoutes.use() - return ( - - ) -} - interface AthenaWorkgroupsProps { bucket: string - workgroup: requests.athena.Workgroup | null } -export default function AthenaWorkgroups({ bucket, workgroup }: AthenaWorkgroupsProps) { - const [prev, setPrev] = React.useState(null) - const data = requests.athena.useWorkgroups(prev) - return data.case({ - Ok: (workgroups) => { - if (!workgroup && workgroups.defaultWorkgroup) - return - return ( -
}> - {workgroups.list.length && ( - - )} -
- ) - }, - Err: (error) => , - _: () => ( +export default function AthenaWorkgroups({ bucket }: AthenaWorkgroupsProps) { + const { queryRun, workgroup, workgroups } = Model.use() + + if (Model.isError(workgroups.data)) return + if (Model.isError(workgroup.data)) return + if (!Model.hasData(workgroups.data) || !Model.hasData(workgroup.data)) { + return ( <> - ), - }) + ) + } + + if (!workgroups.data.list.length) return + + return ( + + ) } diff --git a/catalog/app/containers/Bucket/Queries/Athena/__snapshots__/Database.spec.tsx.snap b/catalog/app/containers/Bucket/Queries/Athena/__snapshots__/Database.spec.tsx.snap new file mode 100644 index 00000000000..f8826f5a665 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/__snapshots__/Database.spec.tsx.snap @@ -0,0 +1,553 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`containers/Bucket/Queries/Athena/Database should disable selection if no spare values 1`] = ` +
+
+ +
+
+ Empty list +
+ + + + +
+
+
+ +
+
+ Empty list +
+ + + + +
+
+
+`; + +exports[`containers/Bucket/Queries/Athena/Database should render selected values 1`] = ` +
+
+ +
+
+ foo +
+ + + + +
+
+
+ +
+
+ bar +
+ + + + +
+
+
+`; + +exports[`containers/Bucket/Queries/Athena/Database should render skeletons 1`] = ` +
+
+
+
+`; + +exports[`containers/Bucket/Queries/Athena/Database should show error when data failed 1`] = ` +
+
+
+ + + +
+
+ Error +
+
+ +
+
+
+
+ + + +
+
+ Error +
+
+ +
+
+
+`; + +exports[`containers/Bucket/Queries/Athena/Database should show error when values failed 1`] = ` +
+
+
+ + + +
+
+ Error +
+
+ +
+
+
+
+ + + +
+
+ Error +
+
+ +
+
+
+`; + +exports[`containers/Bucket/Queries/Athena/Database should show no value (zero-width space) if selected no value 1`] = ` +
+
+ +
+
+ +
+ + + + +
+
+
+ +
+
+ +
+ + + + +
+
+
+`; diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/createPackage.spec.ts b/catalog/app/containers/Bucket/Queries/Athena/model/createPackage.spec.ts new file mode 100644 index 00000000000..8dc33cc15c7 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/createPackage.spec.ts @@ -0,0 +1,213 @@ +import Log from 'utils/Logging' + +import type * as Model from './requests' +import { doQueryResultsContainManifestEntries, parseQueryResults } from './createPackage' + +jest.mock( + 'constants/config', + jest.fn(() => ({})), +) + +describe('containers/Bucket/Queries/Athena/model/createPackage', () => { + describe('parseQueryResults', () => { + it('should return empty', () => { + const results: Model.QueryManifests = { + rows: [], + columns: [], + } + expect(parseQueryResults(results)).toEqual({ + valid: {}, + invalid: [], + }) + }) + + it('should return invalid rows', () => { + const results1: Model.QueryManifests = { + rows: [['s3://foo']], + columns: [{ name: 'physical_key', type: 'varchar' }], + } + const results2: Model.QueryManifests = { + rows: [['s3://foo/a/b/c', 'foo'], ['s3://foo'], ['s3://foo/d/e/f', 'bar', 'baz']], + columns: [ + { name: 'physical_key', type: 'varchar' }, + { name: 'logical_key', type: 'varchar' }, + ], + } + const results3: Model.QueryManifests = { + rows: [['foo', 'bar']], + columns: [ + { name: 'size', type: 'varchar' }, + { name: 'logical_key', type: 'varchar' }, + ], + } + expect(parseQueryResults(results1)).toEqual({ + valid: {}, + invalid: [ + // Not enough columns for a manifest entry + ['s3://foo'], + ], + }) + expect(parseQueryResults(results2)).toEqual({ + valid: { + foo: { + bucket: 'foo', + key: 'a/b/c', + size: 0, + version: undefined, + }, + bar: { + bucket: 'foo', + key: 'd/e/f', + size: 0, + version: undefined, + }, + }, + invalid: [ + // Not enough row elements for a manifest entry + ['s3://foo'], + ], + }) + expect(parseQueryResults(results3)).toEqual({ + valid: {}, + invalid: [ + // Not enough columns for a manifest entry + ['foo', 'bar'], + ], + }) + }) + + it('should return all valid rows', () => { + const results: Model.QueryManifests = { + rows: [ + ['abc', 'a/b/c', '{"a": "b"}', '[s3://a/b/c/d?versionId=def]', '123'], + ['def', 'd/e/f', '{"d": "e"}', '[s3://d/e/f/g?versionId=ghi]', '456', 'extra'], + ['xyz', 'x/y/z', '{"x": "y"}', '[s3://x/y/z/w?versionId=uvw]', '789'], + ], + columns: [ + { name: 'hash', type: 'varchar' }, + { name: 'logical_key', type: 'varchar' }, + { name: 'meta', type: 'varchar' }, + { name: 'physical_keys', type: 'varchar' }, + { name: 'size', type: 'varchar' }, + ], + } + expect(parseQueryResults(results)).toEqual({ + valid: { + 'a/b/c': { + bucket: 'a', + key: 'b/c/d', + size: 123, + version: 'def', + // meta: { a: 'b' }, discarded, not supported for creating packages yet + }, + 'd/e/f': { + bucket: 'd', + key: 'e/f/g', + size: 456, + version: 'ghi', + // meta: { d: 'e' }, discarded, not supported for creating packages yet + }, + 'x/y/z': { + bucket: 'x', + key: 'y/z/w', + size: 789, + version: 'uvw', + // meta: { x: 'y' }, discarded, not supported for creating packages yet + }, + }, + invalid: [], + }) + }) + it('should catch error', () => { + const results: Model.QueryManifests = { + rows: [ + ['abc', 'a/b/c', '{"a": "b"}', '[s3://a/b/c/d?versionId=def]', '123'], + ['def', 'd/e/f', '{"d": "e"}', '[s3://]', '456', 'extra'], + ], + columns: [ + { name: 'hash', type: 'varchar' }, + { name: 'logical_key', type: 'varchar' }, + { name: 'meta', type: 'varchar' }, + { name: 'physical_keys', type: 'varchar' }, + { name: 'size', type: 'varchar' }, + ], + } + const loglevel = Log.getLevel() + Log.setLevel('silent') + expect(parseQueryResults(results)).toEqual({ + valid: { + 'a/b/c': { + bucket: 'a', + key: 'b/c/d', + size: 123, + version: 'def', + // meta: { a: 'b' }, discarded, not supported for creating packages yet + }, + }, + invalid: [['def', 'd/e/f', '{"d": "e"}', '[s3://]', '456', 'extra']], + }) + Log.setLevel(loglevel) + }) + }) + + describe('doQueryResultsContainManifestEntries', () => { + it('does not contain rows', () => { + expect(doQueryResultsContainManifestEntries({ columns: [], rows: [] })).toBe(false) + }) + + it('does not contain valid columns', () => { + expect( + doQueryResultsContainManifestEntries({ + columns: [ + { name: 'foo', type: 'varchar' }, + { name: 'bar', type: 'varchar' }, + ], + rows: [['some']], + }), + ).toBe(false) + }) + + it('does not contain enough columns', () => { + expect( + doQueryResultsContainManifestEntries({ + columns: [ + { name: 'size', type: 'varchar' }, + { name: 'physical_keys', type: 'varchar' }, + ], + rows: [['some']], + }), + ).toBe(false) + expect( + doQueryResultsContainManifestEntries({ + columns: [ + { name: 'size', type: 'varchar' }, + { name: 'physical_key', type: 'varchar' }, + ], + rows: [['some']], + }), + ).toBe(false) + expect( + doQueryResultsContainManifestEntries({ + columns: [ + { name: 'size', type: 'varchar' }, + { name: 'logical_key', type: 'varchar' }, + ], + rows: [['some']], + }), + ).toBe(false) + }) + + it('does contain enough valid data', () => { + expect( + doQueryResultsContainManifestEntries({ + columns: [ + { name: 'size', type: 'varchar' }, + { name: 'physical_key', type: 'varchar' }, + { name: 'logical_key', type: 'varchar' }, + ], + rows: [['some']], + }), + ).toBe(true) + }) + }) +}) diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/createPackage.ts b/catalog/app/containers/Bucket/Queries/Athena/model/createPackage.ts new file mode 100644 index 00000000000..d7f83ebaa7f --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/createPackage.ts @@ -0,0 +1,81 @@ +import type * as Model from 'model' +import * as s3paths from 'utils/s3paths' + +import Log from 'utils/Logging' +import type * as requests from './requests' + +export function doQueryResultsContainManifestEntries( + queryResults: requests.QueryResults, +): queryResults is requests.QueryManifests { + if (!queryResults.rows.length) return false + const columnNames = queryResults.columns.map(({ name }) => name) + return ( + columnNames.includes('size') && + (columnNames.includes('physical_keys') || columnNames.includes('physical_key')) && + columnNames.includes('logical_key') + ) +} + +type Row = requests.QueryManifests['rows'][0] +function parseRow( + row: Row, + columns: requests.QueryResultsColumns, +): { fail?: undefined; ok: [string, Model.S3File] } | { fail: Row; ok?: undefined } { + try { + const entry = row.reduce( + (acc, value, index) => { + if (!columns[index]?.name) return acc + return { + ...acc, + [columns[index].name]: value, + } + }, + {} as Record, + ) + if (!entry.logical_key) return { fail: row } + if (!entry.physical_key && !entry.physical_keys) return { fail: row } + const handle = entry.physical_key + ? s3paths.parseS3Url(entry.physical_key) + : s3paths.parseS3Url(entry.physical_keys.replace(/^\[/, '').replace(/\]$/, '')) + const sizeParsed = Number(entry.size) + const size = Number.isNaN(sizeParsed) ? 0 : sizeParsed + return { + ok: [ + entry.logical_key, + { + ...handle, + size, + }, + ], + } + } catch (e) { + Log.error(e) + return { fail: row } + } +} + +export interface ParsedRows { + valid: Record + invalid: requests.QueryResultsRows +} + +export function parseQueryResults(queryResults: requests.QueryManifests): ParsedRows { + return queryResults.rows + .map((row) => parseRow(row, queryResults.columns)) + .reduce( + (memo, { ok, fail }) => + ok + ? { + valid: { + ...memo.valid, + [ok[0]]: ok[1], + }, + invalid: memo.invalid, + } + : { + valid: memo.valid, + invalid: [...memo.invalid, fail], + }, + { valid: {}, invalid: [] } as ParsedRows, + ) +} diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/index.ts b/catalog/app/containers/Bucket/Queries/Athena/model/index.ts new file mode 100644 index 00000000000..8c441c4a0ab --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/index.ts @@ -0,0 +1,4 @@ +export type * from './requests' +export { NO_DATABASE } from './requests' +export * from './state' +export * from './utils' diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/requests.spec.ts b/catalog/app/containers/Bucket/Queries/Athena/model/requests.spec.ts new file mode 100644 index 00000000000..bf8a0793922 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/requests.spec.ts @@ -0,0 +1,1199 @@ +import type A from 'aws-sdk/clients/athena' +import { act, renderHook } from '@testing-library/react-hooks' + +import Log from 'utils/Logging' + +import * as Model from './utils' +import * as requests from './requests' + +jest.mock( + 'utils/Logging', + jest.fn(() => ({ + error: jest.fn(), + info: jest.fn(), + })), +) + +jest.mock( + 'constants/config', + jest.fn(() => ({})), +) + +const getStorageKey = jest.fn((): string => '') +jest.mock('utils/storage', () => () => ({ + get: jest.fn(() => getStorageKey()), +})) + +function req(output: O, delay = 100) { + return jest.fn((_x: I, callback: (e: Error | null, d: O) => void) => { + const timer = setTimeout(() => { + callback(null, output) + }, delay) + return { + abort: jest.fn(() => { + clearTimeout(timer) + }), + } + }) +} + +function reqThen(output: (x: I) => O, delay = 100) { + return jest.fn((x: I) => ({ + promise: () => + new Promise((resolve) => { + setTimeout(() => { + resolve(output(x)) + }, delay) + }), + })) +} + +const reqThrow = jest.fn(() => ({ + promise: () => { + throw new Error() + }, +})) + +const reqThrowWith = (o: unknown) => + jest.fn(() => ({ + promise: () => { + throw o + }, + })) + +const batchGetQueryExecution = jest.fn() +const getWorkGroup = jest.fn() +const listDataCatalogs = jest.fn() +const listDatabases = jest.fn() +const listQueryExecutions = jest.fn() +const listWorkGroups = jest.fn() +const getQueryExecution = jest.fn() +const listNamedQueries = jest.fn() +const batchGetNamedQuery = jest.fn() +const getQueryResults = jest.fn() +const startQueryExecution = jest.fn() + +jest.mock('utils/AWS', () => ({ + Athena: { + use: () => ({ + batchGetNamedQuery, + batchGetQueryExecution, + getQueryExecution, + getQueryResults, + getWorkGroup, + listDataCatalogs, + listDatabases, + listNamedQueries, + listQueryExecutions, + listWorkGroups, + startQueryExecution, + }), + }, +})) + +describe('containers/Bucket/Queries/Athena/model/requests', () => { + describe('useCatalogNames', () => { + it('return catalog names', async () => { + listDataCatalogs.mockImplementationOnce( + req({ + DataCatalogsSummary: [{ CatalogName: 'foo' }, { CatalogName: 'bar' }], + }), + ) + const { result, waitForNextUpdate } = renderHook(() => requests.useCatalogNames()) + expect(result.current.data).toBe(undefined) + + await act(async () => { + await waitForNextUpdate() + }) + expect(result.current.data).toMatchObject({ list: ['foo', 'bar'] }) + }) + + it('return empty list', async () => { + listDataCatalogs.mockImplementationOnce( + req({ + DataCatalogsSummary: [], + }), + ) + const { result, waitForNextUpdate } = renderHook(() => requests.useCatalogNames()) + + await act(async () => { + await waitForNextUpdate() + }) + expect(result.current.data).toMatchObject({ list: [] }) + }) + + it('return unknowns on invalid data', async () => { + listDataCatalogs.mockImplementationOnce( + req({ + // @ts-expect-error + DataCatalogsSummary: [{ Nonsense: true }, { Absurd: false }], + }), + ) + const { result, waitForNextUpdate } = renderHook(() => requests.useCatalogNames()) + + await act(async () => { + await waitForNextUpdate() + }) + expect(result.current.data).toMatchObject({ list: ['Unknown', 'Unknown'] }) + }) + + it('return empty list on invalid data', async () => { + listDataCatalogs.mockImplementationOnce( + req({ + // @ts-expect-error + Invalid: [], + }), + ) + const { result, waitForNextUpdate } = renderHook(() => requests.useCatalogNames()) + + await act(async () => { + await waitForNextUpdate() + }) + expect(result.current.data).toMatchObject({ list: [] }) + }) + }) + + describe('useCatalogName', () => { + // hooks doesn't support multiple arguments + // https://github.com/testing-library/react-testing-library/issues/1350 + function useWrapper(props: Parameters) { + return requests.useCatalogName(...props) + } + + it('wait for catalog names list', async () => { + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, null] }, + ) + expect(result.current.value).toBe(undefined) + + const error = new Error('Fail') + await act(async () => { + rerender([error, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(error) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + unmount() + }) + + it('switch catalog when execution query loaded', async () => { + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, undefined]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, { catalog: 'bar' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('bar') + unmount() + }) + + it('select execution catalog when catalog list loaded after execution', async () => { + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([Model.Loading, { catalog: 'bar' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(Model.Loading) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, { catalog: 'bar' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('bar') + + unmount() + }) + + it('keep selection when execution has catalog that doesnt exist', async () => { + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, undefined]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, { catalog: 'baz' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + + unmount() + }) + + it('select null when catalog doesnt exist', async () => { + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: [] }, undefined]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(null) + + act(() => { + result.current.setValue('baz') + }) + expect(result.current.value).toBe('baz') + + unmount() + }) + + it('select initial catalog from local storage', async () => { + getStorageKey.mockImplementationOnce(() => 'catalog-bar') + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: ['foo', 'catalog-bar'] }, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('catalog-bar') + + unmount() + }) + }) + + describe('useDatabases', () => { + it('wait for catalogName', async () => { + const { result, rerender, waitForNextUpdate } = renderHook( + (...c: Parameters) => requests.useDatabases(...c), + { + initialProps: undefined, + }, + ) + + await act(async () => { + rerender(Model.Loading) + await waitForNextUpdate() + }) + expect(result.current.data).toBe(Model.Loading) + + const error = new Error('foo') + await act(async () => { + rerender(error) + await waitForNextUpdate() + }) + expect(result.current.data).toBe(error) + }) + + it('return databases', async () => { + listDatabases.mockImplementation( + req({ + DatabaseList: [{ Name: 'bar' }, { Name: 'baz' }], + }), + ) + const { result, waitFor } = renderHook(() => requests.useDatabases('foo')) + + expect((result.all[0] as Model.DataController).data).toBe(undefined) + expect((result.all[1] as Model.DataController).data).toBe(Model.Loading) + await waitFor(() => + expect(result.current.data).toMatchObject({ list: ['bar', 'baz'] }), + ) + }) + + it('handle invalid database', async () => { + listDatabases.mockImplementation( + req({ + // @ts-expect-error + DatabaseList: [{ A: 'B' }, { C: 'D' }], + }), + ) + const { result, waitFor } = renderHook(() => requests.useDatabases('foo')) + await waitFor(() => + expect(result.current.data).toMatchObject({ list: ['Unknown', 'Unknown'] }), + ) + }) + + it('handle invalid list', async () => { + listDatabases.mockImplementation( + req({ + // @ts-expect-error + Foo: 'Bar', + }), + ) + const { result, waitFor } = renderHook(() => requests.useDatabases('foo')) + await waitFor(() => expect(result.current.data).toMatchObject({ list: [] })) + }) + }) + + describe('useDatabase', () => { + function useWrapper(props: Parameters) { + return requests.useDatabase(...props) + } + + it('wait for databases', async () => { + const { result, rerender, waitForNextUpdate, unmount } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, null] }, + ) + expect(result.current.value).toBe(undefined) + + await act(async () => { + rerender([Model.Loading, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(Model.Loading) + + const error = new Error('Fail') + await act(async () => { + rerender([error, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(error) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + + unmount() + }) + + it('switch database when execution query loaded', async () => { + const { result, rerender, waitForNextUpdate, unmount } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, undefined]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, { db: 'bar' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('bar') + + unmount() + }) + + it('select execution db when databases loaded after execution', async () => { + const { result, rerender, waitForNextUpdate, unmount } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([Model.Loading, { db: 'bar' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(Model.Loading) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, { db: 'bar' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('bar') + + unmount() + }) + + it('keep selection when execution has db that doesn’t exist', async () => { + const { result, rerender, waitForNextUpdate, unmount } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, undefined]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, { db: 'baz' }]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + + unmount() + }) + + it('select null when db doesn’t exist', async () => { + const { result, rerender, waitForNextUpdate, unmount } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: [] }, undefined]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe(null) + + act(() => { + result.current.setValue('baz') + }) + expect(result.current.value).toBe('baz') + + unmount() + }) + + it('select initial db from local storage', async () => { + getStorageKey.mockImplementationOnce(() => 'bar') + const { result, rerender, waitForNextUpdate, unmount } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [undefined, undefined] }, + ) + + await act(async () => { + rerender([{ list: ['foo', 'bar'] }, null]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('bar') + + unmount() + }) + }) + + describe('useWorkgroups', () => { + listWorkGroups.mockImplementation( + reqThen(() => ({ + WorkGroups: [{ Name: 'foo' }, { Name: 'bar' }], + })), + ) + + it('return workgroups', async () => { + await act(async () => { + getWorkGroup.mockImplementation( + reqThen(({ WorkGroup: Name }) => ({ + WorkGroup: { + Configuration: { + ResultConfiguration: { + OutputLocation: 'any', + }, + }, + State: 'ENABLED', + Name, + }, + })), + ) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => + expect(result.current.data).toMatchObject({ list: ['bar', 'foo'] }), + ) + unmount() + }) + }) + + it('return only valid workgroups', async () => { + await act(async () => { + getWorkGroup.mockImplementation( + reqThen(({ WorkGroup: Name }) => ({ + WorkGroup: { + Configuration: { + ResultConfiguration: { + OutputLocation: 'any', + }, + }, + State: Name === 'foo' ? 'DISABLED' : 'ENABLED', + Name, + }, + })), + ) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => expect(result.current.data).toMatchObject({ list: ['bar'] })) + unmount() + }) + }) + + it('handle invalid workgroup', async () => { + await act(async () => { + getWorkGroup.mockImplementation( + // @ts-expect-error + reqThen(() => ({ + Invalid: 'foo', + })), + ) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => typeof result.current.data === 'object') + expect(result.current.data).toMatchObject({ list: [] }) + unmount() + }) + }) + + it('handle fail in workgroup', async () => { + await act(async () => { + getWorkGroup.mockImplementation(reqThrow) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => typeof result.current.data === 'object') + expect(Log.error).toBeCalledWith( + 'Fetching "bar" workgroup failed:', + expect.any(Error), + ) + expect(Log.error).toBeCalledWith( + 'Fetching "foo" workgroup failed:', + expect.any(Error), + ) + expect(result.current.data).toMatchObject({ list: [] }) + unmount() + }) + }) + + it('handle access denied for workgroup list', async () => { + await act(async () => { + getWorkGroup.mockImplementation( + reqThrowWith({ + code: 'AccessDeniedException', + }), + ) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => typeof result.current.data === 'object') + expect(Log.info).toBeCalledWith( + 'Fetching "bar" workgroup failed: AccessDeniedException', + ) + expect(Log.info).toBeCalledWith( + 'Fetching "foo" workgroup failed: AccessDeniedException', + ) + expect(result.current.data).toMatchObject({ list: [] }) + unmount() + }) + }) + + it('handle invalid list', async () => { + await act(async () => { + listWorkGroups.mockImplementation( + // @ts-expect-error + reqThen(() => ({ + Invalid: [{ Name: 'foo' }, { Name: 'bar' }], + })), + ) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => typeof result.current.data === 'object') + expect(result.current.data).toMatchObject({ list: [] }) + unmount() + }) + }) + + it('handle no data in list', async () => { + await act(async () => { + listWorkGroups.mockImplementation( + // @ts-expect-error + reqThen(() => null), + ) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => result.current.data instanceof Error) + expect(Log.error).toBeCalledWith( + new TypeError(`Cannot read properties of null (reading 'WorkGroups')`), + ) + expect(result.current.data).toBeInstanceOf(TypeError) + unmount() + }) + }) + + it('handle fail in list', async () => { + await act(async () => { + listWorkGroups.mockImplementation(reqThrow) + const { result, unmount, waitFor } = renderHook(() => requests.useWorkgroups()) + await waitFor(() => result.current.data instanceof Error) + expect(Log.error).toBeCalledWith(expect.any(Error)) + expect(result.current.data).toBeInstanceOf(Error) + unmount() + }) + }) + }) + + describe('useExecutions', () => { + listQueryExecutions.mockImplementation( + req({ + QueryExecutionIds: ['foo', 'bar'], + }), + ) + it('return results', async () => { + batchGetQueryExecution.mockImplementation( + req({ + QueryExecutions: [ + { + QueryExecutionId: '$foo', + }, + { + QueryExecutionId: '$bar', + }, + ], + UnprocessedQueryExecutionIds: [ + { QueryExecutionId: '$baz', ErrorMessage: 'fail' }, + ], + }), + ) + await act(async () => { + const { result, unmount, waitFor } = renderHook(() => + requests.useExecutions('any'), + ) + await waitFor(() => typeof result.current.data === 'object') + expect(result.current.data).toMatchObject({ + list: [ + { id: '$foo' }, + { id: '$bar' }, + { id: '$baz', error: new Error('fail') }, + ], + }) + unmount() + }) + }) + }) + + describe('useWaitForQueryExecution', () => { + it('return execution', async () => { + getQueryExecution.mockImplementation( + req({ + QueryExecution: { QueryExecutionId: '$foo', Status: { State: 'SUCCEEDED' } }, + }), + ) + await act(async () => { + const { result, unmount, waitFor } = renderHook(() => + requests.useWaitForQueryExecution('any'), + ) + await waitFor(() => typeof result.current === 'object') + expect(result.current).toMatchObject({ + id: '$foo', + }) + unmount() + }) + }) + }) + + describe('useQueries', () => { + listNamedQueries.mockImplementation( + req({ + NamedQueryIds: ['foo', 'bar'], + }), + ) + it('return results', async () => { + batchGetNamedQuery.mockImplementation( + req({ + NamedQueries: [ + { + Database: 'any', + QueryString: 'SELECT * FROM *', + NamedQueryId: '$foo', + Name: 'Foo', + }, + { + Database: 'any', + QueryString: 'SELECT * FROM *', + NamedQueryId: '$bar', + Name: 'Bar', + }, + ], + }), + ) + await act(async () => { + const { result, unmount, waitFor } = renderHook(() => requests.useQueries('any')) + await waitFor(() => typeof result.current.data === 'object') + expect(result.current.data).toMatchObject({ + list: [ + { name: 'Bar', key: '$bar', body: 'SELECT * FROM *' }, + { name: 'Foo', key: '$foo', body: 'SELECT * FROM *' }, + ], + }) + unmount() + }) + }) + }) + + describe('useResults', () => { + it('handle empty results', async () => { + getQueryResults.mockImplementation( + req({ + ResultSet: { + Rows: [], + ResultSetMetadata: { + ColumnInfo: [{ Name: 'any', Type: 'some' }], + }, + }, + }), + ) + await act(async () => { + const { result, unmount, waitFor } = renderHook(() => + requests.useResults({ id: 'any' }), + ) + await waitFor(() => typeof result.current.data === 'object') + expect(result.current.data).toMatchObject({ + rows: [], + columns: [], + }) + unmount() + }) + }) + + it('return results', async () => { + getQueryResults.mockImplementation( + req({ + ResultSet: { + Rows: [ + { + Data: [{ VarCharValue: 'foo' }, { VarCharValue: 'bar' }], + }, + { + Data: [{ VarCharValue: 'bar' }, { VarCharValue: 'baz' }], + }, + ], + ResultSetMetadata: { + ColumnInfo: [ + { Name: 'foo', Type: 'some' }, + { Name: 'bar', Type: 'another' }, + ], + }, + }, + }), + ) + await act(async () => { + const { result, unmount, waitFor } = renderHook(() => + requests.useResults({ id: 'any' }), + ) + await waitFor(() => typeof result.current.data === 'object') + expect(result.current.data).toMatchObject({ + rows: [['bar', 'baz']], + columns: [ + { name: 'foo', type: 'some' }, + { name: 'bar', type: 'another' }, + ], + }) + unmount() + }) + }) + }) + + describe('useQueryRun', () => { + it('return execution id', async () => { + startQueryExecution.mockImplementation( + reqThen(() => ({ + QueryExecutionId: 'foo', + })), + ) + await act(async () => { + const { result, unmount, waitForNextUpdate } = renderHook(() => + requests.useQueryRun({ + workgroup: 'a', + catalogName: 'b', + database: 'c', + queryBody: 'd', + }), + ) + await waitForNextUpdate() + const run = await result.current[1](false) + expect(run).toMatchObject({ + id: 'foo', + }) + unmount() + }) + }) + + it('return error if no execution id', async () => { + startQueryExecution.mockImplementation( + reqThen(() => ({})), + ) + await act(async () => { + const { result, unmount, waitForNextUpdate } = renderHook(() => + requests.useQueryRun({ + workgroup: 'a', + catalogName: 'b', + database: 'c', + queryBody: 'd', + }), + ) + await waitForNextUpdate() + const run = await result.current[1](false) + expect(run).toBeInstanceOf(Error) + expect(Log.error).toBeCalledWith(new Error('No execution id')) + if (Model.isError(run)) { + expect(run.message).toBe('No execution id') + } else { + throw new Error('queryRun is not an error') + } + unmount() + }) + }) + + it('handle fail in request', async () => { + startQueryExecution.mockImplementation(reqThrow) + await act(async () => { + const { result, unmount, waitForNextUpdate } = renderHook(() => + requests.useQueryRun({ + workgroup: 'a', + catalogName: 'b', + database: 'c', + queryBody: 'd', + }), + ) + await waitForNextUpdate() + const run = await result.current[1](false) + expect(run).toBeInstanceOf(Error) + unmount() + }) + }) + }) + + describe('useWorkgroup', () => { + function useWrapper(props: Parameters) { + return requests.useWorkgroup(...props) + } + + it('select requested workgroup if it exists', async () => { + await act(async () => { + const workgroups = { + data: { list: ['foo', 'bar'] }, + loadMore: jest.fn(), + } + const { result, waitFor } = renderHook(() => + useWrapper([workgroups, 'bar', undefined]), + ) + await waitFor(() => typeof result.current.data === 'string') + expect(result.current.data).toBe('bar') + }) + }) + + it('select initial workgroup from storage if valid', async () => { + const storageMock = getStorageKey.getMockImplementation() + getStorageKey.mockImplementation(() => 'bar') + const workgroups = { + data: { list: ['foo', 'bar'] }, + loadMore: jest.fn(), + } + + const { result, waitFor, unmount } = renderHook(() => + useWrapper([workgroups, undefined, undefined]), + ) + + await act(async () => { + await waitFor(() => typeof result.current.data === 'string') + expect(result.current.data).toBe('bar') + }) + getStorageKey.mockImplementation(storageMock) + unmount() + }) + + it('select default workgroup from preferences if valid', async () => { + const workgroups = { + data: { list: ['foo', 'bar'] }, + loadMore: jest.fn(), + } + const preferences = { defaultWorkgroup: 'bar' } + + const { result, waitFor, unmount } = renderHook(() => + useWrapper([workgroups, undefined, preferences]), + ) + + await act(async () => { + await waitFor(() => typeof result.current.data === 'string') + expect(result.current.data).toBe('bar') + }) + unmount() + }) + + it('select the first available workgroup if no requested or default', async () => { + await act(async () => { + const workgroups = { + data: { list: ['foo', 'bar', 'baz'] }, + loadMore: jest.fn(), + } + + const { result, waitFor } = renderHook(() => + useWrapper([workgroups, undefined, undefined]), + ) + + await waitFor(() => typeof result.current.data === 'string') + expect(result.current.data).toBe('foo') + }) + }) + + it('return error if no workgroups are available', async () => { + await act(async () => { + const workgroups = { + data: { list: [] }, + loadMore: jest.fn(), + } + + const { result, waitFor } = renderHook(() => + useWrapper([workgroups, undefined, undefined]), + ) + + await waitFor(() => result.current.data instanceof Error) + if (Model.isError(result.current.data)) { + expect(result.current.data.message).toBe('Workgroup not found') + } else { + throw new Error('Not an error') + } + }) + }) + + it('wait for workgroups', async () => { + const workgroups = { + data: undefined, + loadMore: jest.fn(), + } + + const { result, rerender, unmount, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { initialProps: [workgroups, undefined, undefined] }, + ) + expect(result.current.data).toBeUndefined() + + await act(async () => { + rerender() + await waitForNextUpdate() + }) + expect(result.current.data).toBeUndefined() + unmount() + }) + }) + + describe('useQuery', () => { + function useWrapper(props: Parameters) { + return requests.useQuery(...props) + } + + it('sets query to the one matching the execution query', () => { + const queries = { + list: [ + { key: 'foo', name: 'Foo', body: 'SELECT * FROM foo' }, + { key: 'bar', name: 'Bar', body: 'SELECT * FROM bar' }, + ], + } + const execution = { query: 'SELECT * FROM bar' } + const { result } = renderHook(() => useWrapper([queries, execution])) + + if (Model.hasData(result.current.value)) { + expect(result.current.value.body).toBe('SELECT * FROM bar') + } else { + throw new Error('No data') + } + }) + + it('unsets query if no matching execution query', () => { + const queries = { + list: [ + { key: 'foo', name: 'Foo', body: 'SELECT * FROM foo' }, + { key: 'bar', name: 'Bar', body: 'SELECT * FROM bar' }, + ], + } + const execution = { query: 'SELECT * FROM baz' } + const { result } = renderHook(() => useWrapper([queries, execution])) + + if (Model.hasValue(result.current.value)) { + expect(result.current.value).toBe(null) + } else { + throw new Error('No data') + } + }) + + it('sets query to the first one if no execution query is set', () => { + const queries = { + list: [ + { key: 'foo', name: 'Foo', body: 'SELECT * FROM foo' }, + { key: 'bar', name: 'Bar', body: 'SELECT * FROM bar' }, + ], + } + const execution = {} + const { result } = renderHook(() => useWrapper([queries, execution])) + + if (Model.hasData(result.current.value)) { + expect(result.current.value.body).toBe('SELECT * FROM foo') + } else { + throw new Error('No data') + } + }) + + it('sets query to null if no queries are available', () => { + const queries = { list: [] } + const execution = {} + const { result } = renderHook(() => useWrapper([queries, execution])) + + if (Model.hasValue(result.current.value)) { + expect(result.current.value).toBeNull() + } else { + throw new Error('No data') + } + }) + + it('does not change query if a valid query is already selected', async () => { + const queries = { + list: [ + { key: 'foo', name: 'Foo', body: 'SELECT * FROM foo' }, + { key: 'bar', name: 'Bar', body: 'SELECT * FROM bar' }, + ], + } + const execution = { + query: 'SELECT * FROM bar', + } + const { result, rerender, waitForNextUpdate } = renderHook( + (props: Parameters) => useWrapper(props), + { + initialProps: [queries, execution], + }, + ) + + if (Model.hasData(result.current.value)) { + expect(result.current.value.body).toBe('SELECT * FROM bar') + } else { + throw new Error('No data') + } + await act(async () => { + rerender([ + { + list: [ + { key: 'baz', name: 'Baz', body: 'SELECT * FROM baz' }, + { key: 'foo', name: 'Foo', body: 'SELECT * FROM foo' }, + { key: 'bar', name: 'Bar', body: 'SELECT * FROM bar' }, + ], + }, + execution, + ]) + await waitForNextUpdate() + }) + if (Model.hasData(result.current.value)) { + expect(result.current.value.body).toBe('SELECT * FROM bar') + } else { + throw new Error('No data') + } + }) + }) + + describe('useQueryBody', () => { + function useWrapper(props: Parameters) { + return requests.useQueryBody(...props) + } + + it('sets query body from query if query is ready', () => { + const query = { name: 'Foo', key: 'foo', body: 'SELECT * FROM foo' } + const execution = {} + const setQuery = jest.fn() + + const { result } = renderHook(() => useWrapper([query, setQuery, execution])) + + if (Model.hasData(result.current.value)) { + expect(result.current.value).toBe('SELECT * FROM foo') + } else { + throw new Error('No data') + } + }) + + it('sets query body from execution if query is not ready', () => { + const query = null + const execution = { query: 'SELECT * FROM bar' } + const setQuery = jest.fn() + + const { result } = renderHook(() => useWrapper([query, setQuery, execution])) + + if (Model.hasData(result.current.value)) { + expect(result.current.value).toBe('SELECT * FROM bar') + } else { + throw new Error('No data') + } + }) + + it('sets query body to null if query is an error', () => { + const query = new Error('Query failed') + const execution = {} + const setQuery = jest.fn() + + const { result } = renderHook(() => useWrapper([query, setQuery, execution])) + + if (Model.hasValue(result.current.value)) { + expect(result.current.value).toBeNull() + } else { + throw new Error('Unexpected state') + } + }) + + it('does not change value if query and execution are both not ready', async () => { + const query = null + const execution = null + const setQuery = jest.fn() + + const { result, rerender, waitForNextUpdate } = renderHook( + (x: Parameters) => useWrapper(x), + { + initialProps: [query, setQuery, execution], + }, + ) + + expect(result.current.value).toBeUndefined() + act(() => { + result.current.setValue('foo') + }) + expect(result.current.value).toBe('foo') + + await act(async () => { + rerender([query, setQuery, execution]) + await waitForNextUpdate() + }) + expect(result.current.value).toBe('foo') + }) + + it('updates query body and resets query when handleValue is called', async () => { + const query = { name: 'Foo', key: 'foo', body: 'SELECT * FROM foo' } + const execution = {} + const setQuery = jest.fn() + + const { result } = renderHook(() => useWrapper([query, setQuery, execution])) + + act(() => { + result.current.setValue('SELECT * FROM bar') + }) + + expect(result.current.value).toBe('SELECT * FROM bar') + expect(setQuery).toHaveBeenCalledWith(null) + }) + + it('retains value when execution and query are initially empty but later updates', async () => { + const initialQuery = null + const initialExecution = null + const setQuery = jest.fn() + + const { result, rerender, waitForNextUpdate } = renderHook( + (props: Parameters) => useWrapper(props), + { + initialProps: [initialQuery, setQuery, initialExecution], + }, + ) + + expect(result.current.value).toBeUndefined() + + await act(async () => { + rerender([ + { key: 'up', name: 'Updated', body: 'SELECT * FROM updated' }, + setQuery, + initialExecution, + ]) + await waitForNextUpdate() + }) + + if (Model.hasData(result.current.value)) { + expect(result.current.value).toBe('SELECT * FROM updated') + } else { + throw new Error('No data') + } + }) + }) +}) diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/requests.ts b/catalog/app/containers/Bucket/Queries/Athena/model/requests.ts new file mode 100644 index 00000000000..227e6ecbde2 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/requests.ts @@ -0,0 +1,754 @@ +import type Athena from 'aws-sdk/clients/athena' +import * as React from 'react' +import * as Sentry from '@sentry/react' + +import * as AWS from 'utils/AWS' +import * as BucketPreferences from 'utils/BucketPreferences' +import Log from 'utils/Logging' + +import * as storage from './storage' +import * as Model from './utils' + +export interface Query { + // TODO: database? + body: string + description?: string + key: string + name: string +} + +function parseNamedQuery(query: Athena.NamedQuery): Query { + // TODO: database: query.Database! + return { + body: query.QueryString, + description: query.Description, + key: query.NamedQueryId!, + name: query.Name, + } +} + +function listIncludes(list: string[], value: string): boolean { + return list.map((x) => x.toLowerCase()).includes(value.toLowerCase()) +} + +export type Workgroup = string + +interface WorkgroupArgs { + athena: Athena + workgroup: Workgroup +} + +async function fetchWorkgroup({ + athena, + workgroup, +}: WorkgroupArgs): Promise { + try { + const workgroupOutput = await athena.getWorkGroup({ WorkGroup: workgroup }).promise() + if ( + workgroupOutput?.WorkGroup?.Configuration?.ResultConfiguration?.OutputLocation && + workgroupOutput?.WorkGroup?.State === 'ENABLED' && + workgroupOutput?.WorkGroup?.Name + ) { + return workgroupOutput.WorkGroup.Name + } + return null + } catch (error) { + if ((error as $TSFixMe).code === 'AccessDeniedException') { + Log.info(`Fetching "${workgroup}" workgroup failed: ${(error as $TSFixMe).code}`) + } else { + Log.error(`Fetching "${workgroup}" workgroup failed:`, error) + } + return null + } +} + +async function fetchWorkgroups( + athena: Athena, + prev: Model.List | null, +): Promise> { + try { + const workgroupsOutput = await athena + .listWorkGroups({ NextToken: prev?.next }) + .promise() + const parsed = (workgroupsOutput.WorkGroups || []) + .map(({ Name }) => Name || '') + .filter(Boolean) + .sort() + const available = ( + await Promise.all(parsed.map((workgroup) => fetchWorkgroup({ athena, workgroup }))) + ).filter(Boolean) + const list = (prev?.list || []).concat(available as Workgroup[]) + return { + list, + next: workgroupsOutput.NextToken, + } + } catch (e) { + Log.error(e) + throw e + } +} + +export function useWorkgroups(): Model.DataController> { + const athena = AWS.Athena.use() + const [prev, setPrev] = React.useState | null>(null) + const [data, setData] = React.useState>>() + React.useEffect(() => { + let mounted = true + if (!athena) return + fetchWorkgroups(athena, prev) + .then((d) => mounted && setData(d)) + .catch((d) => mounted && setData(d)) + return () => { + mounted = false + } + }, [athena, prev]) + return React.useMemo(() => Model.wrapData(data, setPrev), [data]) +} + +export function useWorkgroup( + workgroups: Model.DataController>, + requestedWorkgroup?: Workgroup, + preferences?: BucketPreferences.AthenaPreferences, +): Model.DataController { + const [data, setData] = React.useState>() + React.useEffect(() => { + if (!Model.hasData(workgroups.data)) return + setData((d) => { + if (!Model.hasData(workgroups.data)) return d + if (requestedWorkgroup && listIncludes(workgroups.data.list, requestedWorkgroup)) { + return requestedWorkgroup + } + const initialWorkgroup = storage.getWorkgroup() || preferences?.defaultWorkgroup + if (initialWorkgroup && listIncludes(workgroups.data.list, initialWorkgroup)) { + return initialWorkgroup + } + return workgroups.data.list[0] || new Error('Workgroup not found') + }) + }, [preferences, requestedWorkgroup, workgroups]) + return React.useMemo( + () => Model.wrapData(data, workgroups.loadMore), + [data, workgroups.loadMore], + ) +} + +export interface QueryExecution { + catalog?: string + completed?: Date + created?: Date + db?: string + id?: string + outputBucket?: string + query?: string + status?: string // 'QUEUED' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'CANCELLED' + workgroup?: Athena.WorkGroupName +} + +export interface QueryExecutionFailed { + id?: string + error: Error +} + +function parseQueryExecution(queryExecution: Athena.QueryExecution): QueryExecution { + return { + catalog: queryExecution?.QueryExecutionContext?.Catalog, + completed: queryExecution?.Status?.CompletionDateTime, + created: queryExecution?.Status?.SubmissionDateTime, + db: queryExecution?.QueryExecutionContext?.Database, + id: queryExecution?.QueryExecutionId, + outputBucket: queryExecution?.ResultConfiguration?.OutputLocation, + query: queryExecution?.Query, + status: queryExecution?.Status?.State, + workgroup: queryExecution?.WorkGroup, + } +} + +function parseQueryExecutionError( + error: Athena.UnprocessedQueryExecutionId, +): QueryExecutionFailed { + return { + error: new Error(error?.ErrorMessage || 'Unknown'), + id: error?.QueryExecutionId, + } +} + +export type QueryExecutionsItem = QueryExecution | QueryExecutionFailed + +export function useExecutions( + workgroup: Model.Data, + queryExecutionId?: string, +): Model.DataController> { + const athena = AWS.Athena.use() + const [prev, setPrev] = React.useState | null>(null) + const [data, setData] = React.useState>>() + + React.useEffect(() => { + if (queryExecutionId) return + if (!Model.hasValue(workgroup)) { + setData(workgroup) + return + } + setData(Model.Loading) + let batchRequest: ReturnType['batchGetQueryExecution']> + + const request = athena?.listQueryExecutions( + { WorkGroup: workgroup, NextToken: prev?.next }, + (error, d) => { + const { QueryExecutionIds, NextToken: next } = d || {} + if (error) { + Sentry.captureException(error) + setData(error) + return + } + if (!QueryExecutionIds || !QueryExecutionIds.length) { + setData({ + list: [], + next, + }) + return + } + batchRequest = athena?.batchGetQueryExecution( + { QueryExecutionIds }, + (batchErr, batchData) => { + const { QueryExecutions, UnprocessedQueryExecutionIds } = batchData || {} + if (batchErr) { + Sentry.captureException(batchErr) + setData(batchErr) + return + } + const parsed = (QueryExecutions || []) + .map(parseQueryExecution) + .concat((UnprocessedQueryExecutionIds || []).map(parseQueryExecutionError)) + const list = (prev?.list || []).concat(parsed) + setData({ + list, + next, + }) + }, + ) + }, + ) + return () => { + request?.abort() + batchRequest?.abort() + } + }, [athena, workgroup, prev, queryExecutionId]) + return React.useMemo(() => Model.wrapData(data, setPrev), [data]) +} + +function useFetchQueryExecution( + QueryExecutionId?: string, +): [Model.Value, () => void] { + const athena = AWS.Athena.use() + const [data, setData] = React.useState>( + QueryExecutionId ? undefined : null, + ) + const [counter, setCounter] = React.useState(0) + React.useEffect(() => { + if (!QueryExecutionId) { + setData(null) + return + } + setData(Model.Loading) + const request = athena?.getQueryExecution({ QueryExecutionId }, (error, d) => { + const { QueryExecution } = d || {} + if (error) { + Sentry.captureException(error) + setData(error) + return + } + const status = QueryExecution?.Status?.State + const parsed = QueryExecution + ? parseQueryExecution(QueryExecution) + : { id: QueryExecutionId } + switch (status) { + case 'FAILED': + case 'CANCELLED': { + const reason = QueryExecution?.Status?.StateChangeReason || '' + setData(new Error(`${status}: ${reason}`)) + break + } + case 'SUCCEEDED': + setData(parsed) + break + case 'QUEUED': + case 'RUNNING': + break + default: + setData(new Error('Unknown query execution status')) + break + } + }) + return () => request?.abort() + }, [athena, QueryExecutionId, counter]) + const fetch = React.useCallback(() => setCounter((prev) => prev + 1), []) + return [data, fetch] +} + +export function useWaitForQueryExecution( + queryExecutionId?: string, +): Model.Value { + const [data, fetch] = useFetchQueryExecution(queryExecutionId) + const [timer, setTimer] = React.useState(null) + React.useEffect(() => { + const t = setInterval(fetch, 1000) + setTimer(t) + return () => clearInterval(t) + }, [queryExecutionId, fetch]) + React.useEffect(() => { + if (Model.isReady(data) && timer) { + clearInterval(timer) + } + }, [timer, data]) + return data +} + +export type QueryResultsValue = Athena.datumString + +interface QueryResultsColumnInfo { + name: T + type: Athena.String +} + +export type QueryResultsColumns = QueryResultsColumnInfo[] +type Row = QueryResultsValue[] +export type QueryResultsRows = Row[] + +export interface QueryResults { + columns: QueryResultsColumns + next?: string + rows: QueryResultsRows +} + +export type ManifestKey = + | 'hash' + | 'logical_key' + | 'meta' + | 'physical_key' + | 'physical_keys' + | 'size' + +export interface QueryManifests extends QueryResults { + columns: QueryResultsColumns +} + +const emptyRow: Row = [] +const emptyList: QueryResultsRows = [] +const emptyColumns: QueryResultsColumns = [] + +export interface QueryRun { + id: string +} + +export type CatalogName = string + +export type Database = string + +export type QueryId = string +export interface QueriesIdsResponse { + list: QueryId[] + next?: string +} + +export function useQueries( + workgroup: Model.Data, +): Model.DataController> { + const athena = AWS.Athena.use() + const [prev, setPrev] = React.useState | null>(null) + const [data, setData] = React.useState>>() + React.useEffect(() => { + if (!Model.hasValue(workgroup)) { + setData(workgroup) + return + } + setData(Model.Loading) + + let batchRequest: ReturnType['batchGetNamedQuery']> + const request = athena?.listNamedQueries( + { + WorkGroup: workgroup, + NextToken: prev?.next, + }, + async (error, d) => { + const { NamedQueryIds, NextToken: next } = d || {} + if (error) { + Sentry.captureException(error) + setData(error) + return + } + if (!NamedQueryIds || !NamedQueryIds.length) { + setData({ + list: prev?.list || [], + next, + }) + return + } + batchRequest = athena?.batchGetNamedQuery( + { NamedQueryIds }, + (batchErr, batchData) => { + const { NamedQueries } = batchData || {} + if (batchErr) { + Sentry.captureException(batchErr) + setData(batchErr) + return + } + const parsed = (NamedQueries || []) + .map(parseNamedQuery) + .sort((a, b) => a.name.localeCompare(b.name)) + const list = (prev?.list || []).concat(parsed) + setData({ + list, + next, + }) + }, + ) + }, + ) + return () => { + request?.abort() + batchRequest?.abort() + } + }, [athena, workgroup, prev]) + return React.useMemo(() => Model.wrapData(data, setPrev), [data]) +} + +export function useResults( + execution: Model.Value, +): Model.DataController { + const athena = AWS.Athena.use() + const [prev, setPrev] = React.useState(null) + const [data, setData] = React.useState>() + + React.useEffect(() => { + if (execution === null) { + setData(undefined) + return + } + if (!Model.hasValue(execution)) { + setData(execution) + return + } + if (!execution.id) { + setData(new Error('Query execution has no ID')) + return + } + + const request = athena?.getQueryResults( + { QueryExecutionId: execution.id, NextToken: prev?.next }, + (error, d) => { + const { ResultSet, NextToken: next } = d || {} + if (error) { + Sentry.captureException(error) + setData(error) + return + } + const parsed = + ResultSet?.Rows?.map( + (row) => row?.Data?.map((item) => item?.VarCharValue || '') || emptyRow, + ) || emptyList + const rows = [...(prev?.rows || emptyList), ...parsed] + if (!rows.length) { + setData({ + rows: [], + columns: [], + next, + }) + return + } + const columns = + ResultSet?.ResultSetMetadata?.ColumnInfo?.map(({ Name, Type }) => ({ + name: Name, + type: Type, + })) || emptyColumns + const isHeadColumns = columns.every(({ name }, index) => name === rows[0][index]) + setData({ + rows: isHeadColumns ? rows.slice(1) : rows, + columns, + next, + }) + }, + ) + return () => request?.abort() + }, [athena, execution, prev]) + return React.useMemo(() => Model.wrapData(data, setPrev), [data]) +} + +export function useDatabases( + catalogName: Model.Value, +): Model.DataController> { + const athena = AWS.Athena.use() + const [prev, setPrev] = React.useState | null>(null) + const [data, setData] = React.useState>>() + React.useEffect(() => { + if (!Model.hasData(catalogName)) { + setData(catalogName || undefined) + return + } + setData(Model.Loading) + const request = athena?.listDatabases( + { + CatalogName: catalogName, + NextToken: prev?.next, + }, + (error, d) => { + const { DatabaseList, NextToken: next } = d || {} + if (error) { + Sentry.captureException(error) + setData(error) + return + } + const list = DatabaseList?.map(({ Name }) => Name || 'Unknown').sort() || [] + setData({ list: (prev?.list || []).concat(list), next }) + }, + ) + return () => request?.abort() + }, [athena, catalogName, prev]) + return React.useMemo(() => Model.wrapData(data, setPrev), [data]) +} + +export function useDatabase( + databases: Model.Data>, + execution: Model.Value, +): Model.ValueController { + const [value, setValue] = React.useState>() + React.useEffect(() => { + if (!Model.hasData(databases)) { + setValue(databases) + return + } + setValue((v) => { + if ( + Model.hasData(execution) && + execution.db && + listIncludes(databases.list, execution.db) + ) { + return execution.db + } + if (Model.hasData(v) && listIncludes(databases.list, v)) { + return v + } + const initialDatabase = storage.getDatabase() + if (initialDatabase && listIncludes(databases.list, initialDatabase)) { + return initialDatabase + } + return databases.list[0] || null + }) + }, [databases, execution]) + return React.useMemo(() => Model.wrapValue(value, setValue), [value]) +} + +export function useCatalogNames(): Model.DataController> { + const athena = AWS.Athena.use() + const [prev, setPrev] = React.useState | null>(null) + const [data, setData] = React.useState>>() + React.useEffect(() => { + const request = athena?.listDataCatalogs({ NextToken: prev?.next }, (error, d) => { + const { DataCatalogsSummary, NextToken: next } = d || {} + setData(Model.Loading) + if (error) { + Sentry.captureException(error) + setData(error) + return + } + const list = DataCatalogsSummary?.map(({ CatalogName }) => CatalogName || 'Unknown') + setData({ + list: (prev?.list || []).concat(list || []), + next, + }) + }) + return () => request?.abort() + }, [athena, prev]) + return React.useMemo(() => Model.wrapData(data, setPrev), [data]) +} + +export function useCatalogName( + catalogNames: Model.Data>, + execution: Model.Value, +): Model.ValueController { + const [value, setValue] = React.useState>() + React.useEffect(() => { + if (!Model.hasData(catalogNames)) { + setValue(catalogNames) + return + } + setValue((v) => { + if ( + Model.hasData(execution) && + execution.catalog && + listIncludes(catalogNames.list, execution.catalog) + ) { + return execution.catalog + } + if (Model.hasData(v) && listIncludes(catalogNames.list, v)) { + return v + } + const initialCatalogName = storage.getCatalog() + if (initialCatalogName && listIncludes(catalogNames.list, initialCatalogName)) { + return initialCatalogName + } + return catalogNames.list[0] || null + }) + }, [catalogNames, execution]) + return React.useMemo(() => Model.wrapValue(value, setValue), [value]) +} + +export function useQuery( + queries: Model.Data>, + execution: Model.Value, +): Model.ValueController { + const [value, setValue] = React.useState>() + React.useEffect(() => { + if (!Model.hasData(queries)) { + setValue(queries) + return + } + setValue((v) => { + if (Model.hasData(execution) && execution.query) { + const executionQuery = queries.list.find((q) => execution.query === q.body) + return executionQuery || null + } + if (Model.hasData(v) && queries.list.includes(v)) { + return v + } + return queries.list[0] || null + }) + }, [execution, queries]) + return React.useMemo(() => Model.wrapValue(value, setValue), [value]) +} + +export function useQueryBody( + query: Model.Value, + setQuery: (value: null) => void, + execution: Model.Value, +): Model.ValueController { + const [value, setValue] = React.useState>() + React.useEffect(() => { + if (!Model.isReady(query)) { + setValue(query) + return + } + setValue((v) => { + if (Model.isError(query)) return null + if (Model.hasData(query)) return query.body + if (Model.hasData(execution) && execution.query) return execution.query + return v + }) + }, [execution, query]) + const handleValue = React.useCallback( + (v: string | null) => { + setQuery(null) + setValue(v) + }, + [setQuery], + ) + return React.useMemo(() => Model.wrapValue(value, handleValue), [value, handleValue]) +} + +export interface ExecutionContext { + catalogName: CatalogName + database: Database +} + +export const NO_DATABASE = new Error('No database') + +interface QueryRunArgs { + workgroup: Model.Data + catalogName: Model.Value + database: Model.Value + queryBody: Model.Value +} + +export function useQueryRun({ + workgroup, + catalogName, + database, + queryBody, +}: QueryRunArgs): [ + Model.Value, + (force: boolean) => Promise>, +] { + const athena = AWS.Athena.use() + // `undefined` = "is not initialized" → is not ready for run + // `null` = is ready but not set, because not submitted for new run + const [value, setValue] = React.useState>() + const prepare = React.useCallback( + (forceDefaultExecutionContext?: boolean) => { + if (!Model.hasData(workgroup)) { + return new Error('No workgroup') + } + + if (!Model.hasValue(catalogName)) { + return catalogName + } + + if (!Model.hasValue(database)) { + return database + } + if (!database && !forceDefaultExecutionContext) { + // We only check if database is selected, + // because if catalogName is not selected, no databases loaded and no database selected as well + return NO_DATABASE + } + + if (!Model.hasData(queryBody)) { + return queryBody + } + return { workgroup, catalogName, database, queryBody } + }, + [workgroup, catalogName, database, queryBody], + ) + React.useEffect(() => { + const init = prepare(true) + setValue(Model.hasData(init) ? null : undefined) + }, [prepare]) + const run = React.useCallback( + async (forceDefaultExecutionContext: boolean) => { + const init = prepare(forceDefaultExecutionContext) + if (!Model.hasData(init)) { + // Error shouldn't be here, because we already checked for errors + // Except `NO_DATABASE`, and if there is some mistake in code + setValue(init) + return init + } + + const options: Athena.Types.StartQueryExecutionInput = { + QueryString: init.queryBody, + ResultConfiguration: { + EncryptionConfiguration: { + EncryptionOption: 'SSE_S3', + }, + }, + WorkGroup: init.workgroup, + } + if (init.catalogName && init.database) { + options.QueryExecutionContext = { + Catalog: init.catalogName, + Database: init.database, + } + } + setValue(Model.Loading) + try { + const d = await athena?.startQueryExecution(options).promise() + const { QueryExecutionId } = d || {} + if (!QueryExecutionId) { + const error = new Error('No execution id') + Log.error(error) + setValue(error) + return error + } + const output = { id: QueryExecutionId } + setValue(output) + return output + } catch (error) { + if (error) { + Log.error(error) + if (error instanceof Error) { + setValue(error) + } + } + return error as Error + } + }, + [athena, prepare], + ) + return [value, run] +} diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/state.spec.tsx b/catalog/app/containers/Bucket/Queries/Athena/model/state.spec.tsx new file mode 100644 index 00000000000..27bfad46eb6 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/state.spec.tsx @@ -0,0 +1,110 @@ +import * as React from 'react' +import renderer from 'react-test-renderer' +import { act, renderHook } from '@testing-library/react-hooks' + +import * as Model from './' + +jest.mock('utils/NamedRoutes', () => ({ + ...jest.requireActual('utils/NamedRoutes'), + use: jest.fn(() => ({ + urls: { + bucketAthenaExecution: () => 'bucket-route', + bucketAthenaWorkgroup: () => 'workgroup-route', + }, + })), +})) + +const useParams = jest.fn( + () => + ({ + bucket: 'b', + workgroup: 'w', + }) as Record, +) + +jest.mock('react-router-dom', () => ({ + ...jest.requireActual('react-router-dom'), + useParams: jest.fn(() => useParams()), + Redirect: jest.fn(() => null), +})) + +const batchGetQueryExecution = jest.fn() +const getWorkGroup = jest.fn() +const listDataCatalogs = jest.fn() +const listDatabases = jest.fn() +const listQueryExecutions = jest.fn() +const listWorkGroups = jest.fn() +const getQueryExecution = jest.fn() +const listNamedQueries = jest.fn() +const batchGetNamedQuery = jest.fn() +const getQueryResults = jest.fn() +const startQueryExecution = jest.fn() + +const AthenaApi = { + batchGetNamedQuery, + batchGetQueryExecution, + getQueryExecution, + getQueryResults, + getWorkGroup, + listDataCatalogs, + listDatabases, + listNamedQueries, + listQueryExecutions, + listWorkGroups, + startQueryExecution, +} + +jest.mock('utils/AWS', () => ({ Athena: { use: () => AthenaApi } })) + +describe('app/containers/Queries/Athena/model/state', () => { + it('throw error when no bucket', () => { + jest.spyOn(console, 'error').mockImplementationOnce(jest.fn()) + useParams.mockImplementationOnce(() => ({})) + const Component = () => { + const state = Model.useState() + return <>{JSON.stringify(state, null, 2)} + } + const tree = () => + renderer.create( + + + , + ) + expect(tree).toThrowError('`bucket` must be defined') + }) + + it('load workgroups and set current workgroup', async () => { + listWorkGroups.mockImplementation(() => ({ + promise: () => + Promise.resolve({ + WorkGroups: [{ Name: 'foo' }, { Name: 'bar' }, { Name: 'w' }], + }), + })) + getWorkGroup.mockImplementation(({ WorkGroup: Name }: { WorkGroup: string }) => ({ + promise: () => + Promise.resolve({ + WorkGroup: { + Configuration: { ResultConfiguration: { OutputLocation: 'any' } }, + State: 'ENABLED', + Name, + }, + }), + })) + listQueryExecutions.mockImplementation((_x, cb) => { + cb(undefined, { QueryExecutionIds: [] }) + return { + abort: jest.fn(), + } + }) + const wrapper = ({ children }: { children: React.ReactNode }) => ( + {children} + ) + const { result, waitFor, unmount } = renderHook(() => Model.useState(), { wrapper }) + await act(async () => { + await waitFor(() => typeof result.current.executions.data === 'object') + }) + expect(result.current.workgroups.data).toMatchObject({ list: ['bar', 'foo', 'w'] }) + expect(result.current.workgroup.data).toBe('w') + unmount() + }) +}) diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/state.tsx b/catalog/app/containers/Bucket/Queries/Athena/model/state.tsx new file mode 100644 index 00000000000..a7bc4631fcb --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/state.tsx @@ -0,0 +1,149 @@ +import invariant from 'invariant' +import * as React from 'react' +import * as RRDom from 'react-router-dom' + +import type * as BucketPreferences from 'utils/BucketPreferences' +import * as NamedRoutes from 'utils/NamedRoutes' + +import * as requests from './requests' +import * as Model from './utils' + +export interface State { + bucket: string + queryExecutionId?: string + + /** + * Query execution loaded by id on the corresponding page. + * On the index page (where there is no queryExecutionId) its value is null. + */ + execution: Model.Value + + /** List of workgroups from Athena */ + workgroups: Model.DataController> + /** + * Workgroup selected by user explicitly or from page URL, and validated that it does exist + * If workgroup doesn't exist, then its value is Error + * It can't be null + */ + workgroup: Model.DataController + /** List of named queries, including query body for each query */ + queries: Model.DataController> + /** Selected named query */ + query: Model.ValueController + /** Query body, typed by user or set from selected named query or query execution */ + queryBody: Model.ValueController + /** List of catalog names from Athena */ + catalogNames: Model.DataController> + /** Catalog name selected by user, or set initially */ + catalogName: Model.ValueController + /** List of databases from Athena */ + databases: Model.DataController> + /** Database selected by user, or set initially */ + database: Model.ValueController + /** List of query executions, in other words, history of executions */ + executions: Model.DataController> + /** Rows and columns of query results */ + results: Model.DataController + + /** + * Submit query to Athena with values memoized here in state + * If catalog name or database is not selected, then it will return specific output + * Which is handled and then user can re-submit with `forceDefaultExecutionContext: true` + */ + submit: ( + forceDefaultExecutionContext: boolean, + ) => Promise> + /** + * Query run is `undefined` when there is not enough data to run the query + * It is `null` when it is ready to run + * Error when submit failed or when validation failed (e.g. no database selected) + */ + queryRun: Model.Value +} + +export const Ctx = React.createContext(null) + +interface ProviderProps { + preferences?: BucketPreferences.AthenaPreferences + children: React.ReactNode +} + +export function Provider({ preferences, children }: ProviderProps) { + const { urls } = NamedRoutes.use() + + const { + bucket, + queryExecutionId, + workgroup: workgroupId, + } = RRDom.useParams<{ + bucket: string + queryExecutionId?: string + workgroup?: requests.Workgroup + }>() + invariant(!!bucket, '`bucket` must be defined') + + const execution = requests.useWaitForQueryExecution(queryExecutionId) + + const workgroups = requests.useWorkgroups() + const workgroup = requests.useWorkgroup(workgroups, workgroupId, preferences) + const queries = requests.useQueries(workgroup.data) + const query = requests.useQuery(queries.data, execution) + const queryBody = requests.useQueryBody(query.value, query.setValue, execution) + const catalogNames = requests.useCatalogNames() + const catalogName = requests.useCatalogName(catalogNames.data, execution) + const databases = requests.useDatabases(catalogName.value) + const database = requests.useDatabase(databases.data, execution) + const executions = requests.useExecutions(workgroup.data, queryExecutionId) + const results = requests.useResults(execution) + + const [queryRun, submit] = requests.useQueryRun({ + workgroup: workgroup.data, + catalogName: catalogName.value, + database: database.value, + queryBody: queryBody.value, + }) + + const value: State = { + bucket, + queryExecutionId, + workgroup, + + catalogName, + catalogNames, + database, + databases, + execution, + executions, + queries, + query, + queryBody, + results, + workgroups, + + submit, + queryRun, + } + + if (Model.hasData(queryRun) && queryExecutionId !== queryRun.id) { + return ( + + ) + } + + if (Model.hasData(workgroup.data) && !workgroupId) { + return + } + + return {children} +} + +/** state object is not memoized, use destructuring down to memoized properties */ +export function useState() { + const model = React.useContext(Ctx) + invariant(model, 'Athena state accessed outside of provider') + return model +} + +export const use = useState diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/storage.ts b/catalog/app/containers/Bucket/Queries/Athena/model/storage.ts new file mode 100644 index 00000000000..1c1fd07d5bc --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/storage.ts @@ -0,0 +1,28 @@ +import mkStorage from 'utils/storage' + +const ATHENA_WORKGROUP_KEY = 'ATHENA_WORKGROUP' + +const ATHENA_CATALOG_KEY = 'ATHENA_CATALOG' + +const ATHENA_DATABASE_KEY = 'ATHENA_DATABASE' + +const storage = mkStorage({ + athenaCatalog: ATHENA_CATALOG_KEY, + athenaDatabase: ATHENA_DATABASE_KEY, + athenaWorkgroup: ATHENA_WORKGROUP_KEY, +}) + +export const getCatalog = () => storage.get('athenaCatalog') + +export const setCatalog = (catalog: string) => storage.set('athenaCatalog', catalog) + +export const getDatabase = () => storage.get('athenaDatabase') + +export const setDatabase = (database: string) => storage.set('athenaDatabase', database) + +export const clearDatabase = () => storage.remove('athenaDatabase') + +export const getWorkgroup = () => storage.get('athenaWorkgroup') + +export const setWorkgroup = (workgroup: string) => + storage.set('athenaWorkgroup', workgroup) diff --git a/catalog/app/containers/Bucket/Queries/Athena/model/utils.ts b/catalog/app/containers/Bucket/Queries/Athena/model/utils.ts new file mode 100644 index 00000000000..429b8f144d3 --- /dev/null +++ b/catalog/app/containers/Bucket/Queries/Athena/model/utils.ts @@ -0,0 +1,96 @@ +export const Loading = Symbol('loading') + +export type Maybe = T | null + +// `T` is the value +// `undefined` is no data. It is not initialized +// `Loading` is loading +// `Error` is error +export type Data = T | undefined | typeof Loading | Error + +export interface DataController { + data: Data + loadMore: () => void +} + +export function wrapData(data: Data, setPrev: (d: T) => void): DataController { + return { + data, + loadMore: () => hasData(data) && setPrev(data), + } +} + +export interface List { + list: T[] + next?: string +} + +// `T` is the value +// `null` is no value, explicitly set by user +// `undefined` is no value. It is not initialized +// `Loading` is loading +// `Error` is error +export type Value = Maybe> + +export interface ValueController { + value: Value + setValue: (v: T | null) => void +} + +export function wrapValue( + value: Value, + setValue: (d: T | null) => void, +): ValueController { + return { + value, + setValue, + } +} + +/** Data is loaded, or value is set to actual value */ +export function hasData(value: Value): value is T { + if ( + value === undefined || + value === Loading || + value instanceof Error || + value === null + ) { + return false + } + return true +} + +/** No value yet: value or data was just initialized */ +export function isNone(value: Value): value is undefined { + return value === undefined +} + +/** Data is loading, or value is waiting for data */ +export function isLoading(value: Value): value is typeof Loading { + return value === Loading +} + +export function isError(value: Value): value is Error { + return value instanceof Error +} + +/** Value is selected with some or no value, or resolved with error, or data is loaded (successfully or not) */ +export function isReady(value: Value): value is T | null | Error { + if (value === undefined || value === Loading) { + return false + } + return true +} + +/** Value is selected with some or no value, or data is loaded successfully */ +export function hasValue(value: Value): value is T | null { + if (value === undefined || value === Loading || value instanceof Error) { + return false + } + return true +} + +/** User explicitly set no value */ +export function isNoneSelected(value: Value): value is null { + return value === null +} diff --git a/catalog/app/containers/Bucket/Queries/ElasticSearch.tsx b/catalog/app/containers/Bucket/Queries/ElasticSearch.tsx index c9d6d23d071..03b793435fa 100644 --- a/catalog/app/containers/Bucket/Queries/ElasticSearch.tsx +++ b/catalog/app/containers/Bucket/Queries/ElasticSearch.tsx @@ -24,9 +24,6 @@ const useStyles = M.makeStyles((t) => ({ form: { margin: t.spacing(0, 0, 4), }, - sectionHeader: { - margin: t.spacing(0, 0, 1), - }, select: { margin: t.spacing(3, 0), }, @@ -62,7 +59,7 @@ interface QueriesStateRenderProps { error: Error | null handleError: (error: Error | null) => void handleQueryBodyChange: (q: requests.ElasticSearchQuery | null) => void - handleQueryMetaChange: (q: requests.Query | requests.athena.AthenaQuery | null) => void + handleQueryMetaChange: (q: requests.Query | requests.athena.Query | null) => void handleSubmit: (q: requests.ElasticSearchQuery) => () => void queries: requests.Query[] queryData: requests.AsyncData @@ -99,7 +96,7 @@ function QueriesState({ bucket, children }: QueriesStateProps) { ) const handleQueryMetaChange = React.useCallback( - (q: requests.athena.AthenaQuery | requests.Query | null) => { + (q: requests.athena.Query | requests.Query | null) => { setQueryMeta(q as requests.Query | null) setCustomQueryBody(null) }, @@ -212,10 +209,8 @@ export default function ElastiSearch() { ElasticSearch queries
- - Select query - + label="Select query" queries={queries} onChange={handleQueryMetaChange} value={customQueryBody ? null : queryMeta} diff --git a/catalog/app/containers/Bucket/Queries/QuerySelect.spec.tsx b/catalog/app/containers/Bucket/Queries/QuerySelect.spec.tsx index 61425326cd4..f681ed82b50 100644 --- a/catalog/app/containers/Bucket/Queries/QuerySelect.spec.tsx +++ b/catalog/app/containers/Bucket/Queries/QuerySelect.spec.tsx @@ -6,7 +6,7 @@ import QuerySelect from './QuerySelect' describe('containers/Bucket/Queries/QuerySelect', () => { it('should render', () => { const tree = renderer - .create( {}} value={null} />) + .create( {}} value={null} />) .toJSON() expect(tree).toMatchSnapshot() }) @@ -16,7 +16,14 @@ describe('containers/Bucket/Queries/QuerySelect', () => { { key: 'key2', name: 'name2', url: 'url2' }, ] const tree = renderer - .create( {}} value={queries[1]} />) + .create( + {}} + value={queries[1]} + />, + ) .toJSON() expect(tree).toMatchSnapshot() }) diff --git a/catalog/app/containers/Bucket/Queries/QuerySelect.tsx b/catalog/app/containers/Bucket/Queries/QuerySelect.tsx index 38bb51218bc..d0f30c71684 100644 --- a/catalog/app/containers/Bucket/Queries/QuerySelect.tsx +++ b/catalog/app/containers/Bucket/Queries/QuerySelect.tsx @@ -8,34 +8,26 @@ interface AbstractQuery { } interface QuerySelectProps { + className?: string + disabled?: boolean + label: React.ReactNode onChange: (value: T | null) => void onLoadMore?: () => void queries: T[] value: T | null } -const useStyles = M.makeStyles((t) => ({ - header: { - margin: t.spacing(0, 0, 1), - }, - selectWrapper: { - width: '100%', - }, - select: { - padding: t.spacing(1), - }, -})) - const LOAD_MORE = 'load-more' export default function QuerySelect({ - queries, + className, + disabled, + label, onChange, onLoadMore, + queries, value, }: QuerySelectProps) { - const classes = useStyles() - const handleChange = React.useCallback( (event) => { if (event.target.value === LOAD_MORE && onLoadMore) { @@ -48,31 +40,29 @@ export default function QuerySelect({ ) return ( - - - - - Custom + + {label} + + + Custom + + {queries.map((query) => ( + + + + ))} + {!!onLoadMore && ( + + + Load more + - {queries.map((query) => ( - - - - ))} - {!!onLoadMore && ( - - - Load more - - - )} - - - + )} + + ) } diff --git a/catalog/app/containers/Bucket/Queries/__snapshots__/QuerySelect.spec.tsx.snap b/catalog/app/containers/Bucket/Queries/__snapshots__/QuerySelect.spec.tsx.snap index 03981390857..ff49164ebb1 100644 --- a/catalog/app/containers/Bucket/Queries/__snapshots__/QuerySelect.spec.tsx.snap +++ b/catalog/app/containers/Bucket/Queries/__snapshots__/QuerySelect.spec.tsx.snap @@ -2,111 +2,115 @@ exports[`containers/Bucket/Queries/QuerySelect should render 1`] = `
+
-
- - Custom - -
+ Custom +
- - - -
+ + + +
`; exports[`containers/Bucket/Queries/QuerySelect should render with selected value 1`] = `
+
-
- - name2 - -
+ name2 +
- - - -
+ + + +
`; diff --git a/catalog/app/containers/Bucket/Queries/requests/athena.ts b/catalog/app/containers/Bucket/Queries/requests/athena.ts deleted file mode 100644 index 135976b574a..00000000000 --- a/catalog/app/containers/Bucket/Queries/requests/athena.ts +++ /dev/null @@ -1,582 +0,0 @@ -import Athena from 'aws-sdk/clients/athena' -import * as React from 'react' - -import * as AWS from 'utils/AWS' -import * as BucketPreferences from 'utils/BucketPreferences' -import { useData } from 'utils/Data' -import wait from 'utils/wait' - -import * as storage from './storage' - -import { AsyncData } from './requests' - -// TODO: rename to requests.athena.Query -export interface AthenaQuery { - body: string - description?: string - key: string - name: string -} - -export interface QueriesResponse { - list: AthenaQuery[] - next?: string -} - -interface QueriesArgs { - athena: Athena - prev: QueriesResponse | null - workgroup: string -} - -function parseNamedQuery(query: Athena.NamedQuery): AthenaQuery { - return { - body: query.QueryString, - description: query.Description, - key: query.NamedQueryId!, - name: query.Name, - } -} - -async function fetchQueries({ - athena, - prev, - workgroup, -}: QueriesArgs): Promise { - try { - const queryIdsOutput = await athena - ?.listNamedQueries({ WorkGroup: workgroup, NextToken: prev?.next }) - .promise() - if (!queryIdsOutput.NamedQueryIds || !queryIdsOutput.NamedQueryIds.length) - return { - list: prev?.list || [], - next: queryIdsOutput.NextToken, - } - - const queriesOutput = await athena - ?.batchGetNamedQuery({ - NamedQueryIds: queryIdsOutput.NamedQueryIds, - }) - .promise() - const parsed = (queriesOutput.NamedQueries || []).map(parseNamedQuery) - const list = (prev?.list || []).concat(parsed) - return { - list, - next: queryIdsOutput.NextToken, - } - } catch (e) { - // eslint-disable-next-line no-console - console.log('Unable to fetch') - // eslint-disable-next-line no-console - console.error(e) - throw e - } -} - -export function useQueries( - workgroup: string, - prev: QueriesResponse | null, -): AsyncData { - const athena = AWS.Athena.use() - return useData(fetchQueries, { athena, prev, workgroup }, { noAutoFetch: !workgroup }) -} - -export type Workgroup = string - -function getDefaultWorkgroup( - list: Workgroup[], - preferences?: BucketPreferences.AthenaPreferences, -): Workgroup { - const workgroupFromConfig = preferences?.defaultWorkgroup - if (workgroupFromConfig && list.includes(workgroupFromConfig)) { - return workgroupFromConfig - } - return storage.getWorkgroup() || list[0] -} - -interface WorkgroupArgs { - athena: Athena - workgroup: Workgroup -} - -async function fetchWorkgroup({ - athena, - workgroup, -}: WorkgroupArgs): Promise { - try { - const workgroupOutput = await athena.getWorkGroup({ WorkGroup: workgroup }).promise() - if ( - workgroupOutput?.WorkGroup?.Configuration?.ResultConfiguration?.OutputLocation && - workgroupOutput?.WorkGroup?.State === 'ENABLED' && - workgroupOutput?.WorkGroup?.Name - ) { - return workgroupOutput.WorkGroup.Name - } - return null - } catch (error) { - return null - } -} - -export interface WorkgroupsResponse { - defaultWorkgroup: Workgroup - list: Workgroup[] - next?: string -} - -interface WorkgroupsArgs { - athena: Athena - prev: WorkgroupsResponse | null - preferences?: BucketPreferences.AthenaPreferences -} - -async function fetchWorkgroups({ - athena, - prev, - preferences, -}: WorkgroupsArgs): Promise { - try { - const workgroupsOutput = await athena - .listWorkGroups({ NextToken: prev?.next }) - .promise() - const parsed = (workgroupsOutput.WorkGroups || []).map( - ({ Name }) => Name || 'Unknown', - ) - const available = ( - await Promise.all(parsed.map((workgroup) => fetchWorkgroup({ athena, workgroup }))) - ).filter(Boolean) - const list = (prev?.list || []).concat(available as Workgroup[]) - return { - defaultWorkgroup: getDefaultWorkgroup(list, preferences), - list, - next: workgroupsOutput.NextToken, - } - } catch (e) { - // eslint-disable-next-line no-console - console.log('Unable to fetch') - // eslint-disable-next-line no-console - console.error(e) - throw e - } -} - -export function useWorkgroups( - prev: WorkgroupsResponse | null, -): AsyncData { - const athena = AWS.Athena.use() - const prefs = BucketPreferences.use() - const preferences = React.useMemo( - () => - BucketPreferences.Result.match( - { - Ok: ({ ui }) => ui.athena, - _: () => undefined, - }, - prefs, - ), - [prefs], - ) - return useData(fetchWorkgroups, { athena, prev, preferences }) -} - -export interface QueryExecution { - catalog?: string - completed?: Date - created?: Date - db?: string - error?: Error - id?: string - outputBucket?: string - query?: string - status?: string // 'QUEUED' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'CANCELLED' - workgroup?: Athena.WorkGroupName -} - -export interface QueryExecutionsResponse { - list: QueryExecution[] - next?: string -} - -interface QueryExecutionsArgs { - athena: Athena - prev: QueryExecutionsResponse | null - workgroup: string -} - -function parseQueryExecution(queryExecution: Athena.QueryExecution): QueryExecution { - return { - catalog: queryExecution?.QueryExecutionContext?.Catalog, - completed: queryExecution?.Status?.CompletionDateTime, - created: queryExecution?.Status?.SubmissionDateTime, - db: queryExecution?.QueryExecutionContext?.Database, - id: queryExecution?.QueryExecutionId, - outputBucket: queryExecution?.ResultConfiguration?.OutputLocation, - query: queryExecution?.Query, - status: queryExecution?.Status?.State, - workgroup: queryExecution?.WorkGroup, - } -} - -function parseQueryExecutionError( - error: Athena.UnprocessedQueryExecutionId, -): QueryExecution { - return { - error: new Error(error?.ErrorMessage || 'Unknown'), - id: error?.QueryExecutionId, - } -} - -async function fetchQueryExecutions({ - athena, - prev, - workgroup, -}: QueryExecutionsArgs): Promise { - try { - const executionIdsOutput = await athena - .listQueryExecutions({ WorkGroup: workgroup, NextToken: prev?.next }) - .promise() - - const ids = executionIdsOutput.QueryExecutionIds - if (!ids || !ids.length) - return { - list: [], - next: executionIdsOutput.NextToken, - } - - const executionsOutput = await athena - ?.batchGetQueryExecution({ QueryExecutionIds: ids }) - .promise() - const parsed = (executionsOutput.QueryExecutions || []) - .map(parseQueryExecution) - .concat( - (executionsOutput.UnprocessedQueryExecutionIds || []).map( - parseQueryExecutionError, - ), - ) - const list = (prev?.list || []).concat(parsed) - return { - list, - next: executionIdsOutput.NextToken, - } - } catch (e) { - // eslint-disable-next-line no-console - console.log('Unable to fetch') - // eslint-disable-next-line no-console - console.error(e) - throw e - } -} - -export function useQueryExecutions( - workgroup: string, - prev: QueryExecutionsResponse | null, -): AsyncData { - const athena = AWS.Athena.use() - return useData( - fetchQueryExecutions, - { athena, prev, workgroup }, - { noAutoFetch: !workgroup }, - ) -} - -async function waitForQueryStatus( - athena: Athena, - QueryExecutionId: string, -): Promise { - // eslint-disable-next-line no-constant-condition - while (true) { - // NOTE: await is used to intentionally pause loop and make requests in series - // eslint-disable-next-line no-await-in-loop - const statusData = await athena.getQueryExecution({ QueryExecutionId }).promise() - const status = statusData?.QueryExecution?.Status?.State - const parsed = statusData?.QueryExecution - ? parseQueryExecution(statusData?.QueryExecution) - : { - id: QueryExecutionId, - } - if (status === 'FAILED' || status === 'CANCELLED') { - const reason = statusData?.QueryExecution?.Status?.StateChangeReason || '' - return { - ...parsed, - error: new Error(`${status}: ${reason}`), - } - } - - if (!status) { - return { - ...parsed, - error: new Error('Unknown query execution status'), - } - } - - if (status === 'SUCCEEDED') { - return parsed - } - - // eslint-disable-next-line no-await-in-loop - await wait(1000) - } -} - -export type QueryResultsValue = Athena.datumString - -export interface QueryResultsColumnInfo { - name: Athena.String - type: Athena.String -} - -export type QueryResultsColumns = QueryResultsColumnInfo[] -type Row = QueryResultsValue[] -export type QueryResultsRows = Row[] - -export interface QueryResultsResponse { - columns: QueryResultsColumns - next?: string - queryExecution: QueryExecution - rows: QueryResultsRows -} - -type ManifestKey = 'hash' | 'logical_key' | 'meta' | 'physical_keys' | 'size' - -export interface QueryManifestsResponse extends QueryResultsResponse { - rows: [ManifestKey[], ...string[][]] -} - -interface QueryResultsArgs { - athena: Athena - queryExecutionId: string - prev: QueryResultsResponse | null -} - -const emptyRow: Row = [] -const emptyList: QueryResultsRows = [] -const emptyColumns: QueryResultsColumns = [] - -async function fetchQueryResults({ - athena, - queryExecutionId, - prev, -}: QueryResultsArgs): Promise { - const queryExecution = await waitForQueryStatus(athena, queryExecutionId) - if (queryExecution.error) { - return { - rows: emptyList, - columns: emptyColumns, - queryExecution, - } - } - - try { - const queryResultsOutput = await athena - .getQueryResults({ - QueryExecutionId: queryExecutionId, - NextToken: prev?.next, - }) - .promise() - const parsed = - queryResultsOutput.ResultSet?.Rows?.map( - (row) => row?.Data?.map((item) => item?.VarCharValue || '') || emptyRow, - ) || emptyList - const rows = [...(prev?.rows || emptyList), ...parsed] - const columns = - queryResultsOutput.ResultSet?.ResultSetMetadata?.ColumnInfo?.map( - ({ Name, Type }) => ({ - name: Name, - type: Type, - }), - ) || emptyColumns - const isHeadColumns = columns.every(({ name }, index) => name === rows[0][index]) - return { - rows: isHeadColumns ? rows.slice(1) : rows, - columns, - next: queryResultsOutput.NextToken, - queryExecution, - } - } catch (error) { - return { - rows: emptyList, - columns: emptyColumns, - queryExecution: { - ...queryExecution, - error: error instanceof Error ? error : new Error(`${error}`), - }, - } - } -} - -export function useQueryResults( - queryExecutionId: string | null, - prev: QueryResultsResponse | null, -): AsyncData { - const athena = AWS.Athena.use() - return useData( - fetchQueryResults, - { athena, prev, queryExecutionId }, - { noAutoFetch: !queryExecutionId }, - ) -} - -export interface QueryRunResponse { - id: string -} - -export type CatalogName = string -export interface CatalogNamesResponse { - list: CatalogName[] - next?: string -} - -interface CatalogNamesArgs { - athena: Athena - prev?: CatalogNamesResponse -} - -async function fetchCatalogNames({ - athena, - prev, -}: CatalogNamesArgs): Promise { - const catalogsOutput = await athena - ?.listDataCatalogs({ NextToken: prev?.next }) - .promise() - const list = - catalogsOutput?.DataCatalogsSummary?.map( - ({ CatalogName }) => CatalogName || 'Unknown', - ) || [] - return { - list: (prev?.list || []).concat(list), - next: catalogsOutput.NextToken, - } -} - -export function useCatalogNames( - prev: CatalogNamesResponse | null, -): AsyncData { - const athena = AWS.Athena.use() - return useData(fetchCatalogNames, { athena, prev }) -} - -export type Database = string -export interface DatabasesResponse { - list: CatalogName[] - next?: string -} - -interface DatabasesArgs { - athena: Athena - catalogName: CatalogName - prev?: DatabasesResponse -} - -async function fetchDatabases({ - athena, - catalogName, - prev, -}: DatabasesArgs): Promise { - const databasesOutput = await athena - ?.listDatabases({ CatalogName: catalogName, NextToken: prev?.next }) - .promise() - // TODO: add `Description` besides `Name` - const list = databasesOutput?.DatabaseList?.map(({ Name }) => Name || 'Unknown') || [] - return { - list: (prev?.list || []).concat(list), - next: databasesOutput.NextToken, - } -} - -export function useDatabases( - catalogName: CatalogName | null, - prev: DatabasesResponse | null, -): AsyncData { - const athena = AWS.Athena.use() - return useData( - fetchDatabases, - { athena, catalogName, prev }, - { noAutoFetch: !catalogName }, - ) -} - -interface DefaultDatabaseArgs { - athena: Athena -} - -async function fetchDefaultQueryExecution({ - athena, -}: DefaultDatabaseArgs): Promise { - const catalogNames = await fetchCatalogNames({ athena }) - if (!catalogNames.list.length) { - return null - } - const catalogName = catalogNames.list[0] - const databases = await fetchDatabases({ athena, catalogName }) - if (!databases.list.length) { - return null - } - return { - catalog: catalogName, - db: databases.list[0], - } -} - -export function useDefaultQueryExecution(): AsyncData { - const athena = AWS.Athena.use() - return useData(fetchDefaultQueryExecution, { athena }) -} - -export interface ExecutionContext { - catalogName: CatalogName - database: Database -} - -interface RunQueryArgs { - athena: Athena - queryBody: string - workgroup: string - executionContext: ExecutionContext | null -} - -export async function runQuery({ - athena, - queryBody, - workgroup, - executionContext, -}: RunQueryArgs): Promise { - try { - const options: Athena.Types.StartQueryExecutionInput = { - QueryString: queryBody, - ResultConfiguration: { - EncryptionConfiguration: { - EncryptionOption: 'SSE_S3', - }, - }, - WorkGroup: workgroup, - } - if (executionContext) { - options.QueryExecutionContext = { - Catalog: executionContext.catalogName, - Database: executionContext.database, - } - } - const { QueryExecutionId } = await athena.startQueryExecution(options).promise() - if (!QueryExecutionId) throw new Error('No execution id') - return { - id: QueryExecutionId, - } - } catch (e) { - // eslint-disable-next-line no-console - console.log('Unable to fetch') - // eslint-disable-next-line no-console - console.error(e) - throw e - } -} - -export function useQueryRun(workgroup: string) { - const athena = AWS.Athena.use() - return React.useCallback( - (queryBody: string, executionContext: ExecutionContext | null) => { - if (!athena) return Promise.reject(new Error('No Athena available')) - return runQuery({ athena, queryBody, workgroup, executionContext }) - }, - [athena, workgroup], - ) -} diff --git a/catalog/app/containers/Bucket/Queries/requests/index.ts b/catalog/app/containers/Bucket/Queries/requests/index.ts index 0dc2f97cd35..b02355f69bb 100644 --- a/catalog/app/containers/Bucket/Queries/requests/index.ts +++ b/catalog/app/containers/Bucket/Queries/requests/index.ts @@ -1,4 +1,4 @@ -export * as athena from './athena' +export type * as athena from '../Athena/model/requests' export * from './queriesConfig' diff --git a/catalog/app/containers/Bucket/Queries/requests/storage.ts b/catalog/app/containers/Bucket/Queries/requests/storage.ts deleted file mode 100644 index 5cbcc8d7b64..00000000000 --- a/catalog/app/containers/Bucket/Queries/requests/storage.ts +++ /dev/null @@ -1,10 +0,0 @@ -import mkStorage from 'utils/storage' - -const ATHENA_WORKGROUP_KEY = 'ATHENA_WORKGROUP' - -const storage = mkStorage({ athenaWorkgroup: ATHENA_WORKGROUP_KEY }) - -export const getWorkgroup = () => storage.get('athenaWorkgroup') - -export const setWorkgroup = (workgroup: string) => - storage.set('athenaWorkgroup', workgroup) diff --git a/catalog/app/utils/AWS/Bedrock/History.spec.ts b/catalog/app/utils/AWS/Bedrock/History.spec.ts index 6ff1fb4773a..d635295b0b9 100644 --- a/catalog/app/utils/AWS/Bedrock/History.spec.ts +++ b/catalog/app/utils/AWS/Bedrock/History.spec.ts @@ -21,7 +21,7 @@ describe('utils/AWS/Bedrock/History', () => { }) describe('foldMessages', () => { - it('Fold same-role messages', async () => { + it('Fold same-role messages', () => { const userFoo = Message.createMessage('foo') const userBar = Message.createMessage('bar') const assistantFoo = Message.createMessage('foo', 'assistant') @@ -32,7 +32,7 @@ describe('utils/AWS/Bedrock/History', () => { expect(list[1].content).toBe('foo\nbaz') }) - it('Fold system and user messages', async () => { + it('Fold system and user messages', () => { const userFoo = Message.createMessage('foo') const userBar = Message.createMessage('bar') const systemFoo = Message.createMessage('foo', 'system') diff --git a/catalog/app/utils/Sentry.ts b/catalog/app/utils/Sentry.ts index 02b99626958..6ca5da74976 100644 --- a/catalog/app/utils/Sentry.ts +++ b/catalog/app/utils/Sentry.ts @@ -60,13 +60,13 @@ export const UserTracker = function SentryUserTracker({ return children } -/** @deprecated */ +/** @deprecated use '@sentry/react' */ async function callSentry(method: string, ...args: $TSFixMe[]) { return (Sentry as $TSFixMe)[method](...args) } -/** @deprecated */ +/** @deprecated use '@sentry/react' */ export const useSentry = () => callSentry -/** @deprecated */ +/** @deprecated use '@sentry/react' */ export const use = useSentry