diff --git a/catalog/CHANGELOG.md b/catalog/CHANGELOG.md index 5409223ec66..546d0b49d6f 100644 --- a/catalog/CHANGELOG.md +++ b/catalog/CHANGELOG.md @@ -17,6 +17,7 @@ where verb is one of ## Changes +- [Added] Admin: Tabulator Settings (open query) ([#4255](https://github.com/quiltdata/quilt/pull/4255)) - [Added] Visual editor for `quilt_summarize.json` ([#4254](https://github.com/quiltdata/quilt/pull/4254)) - [Added] Support "html" type in `quilt_summarize.json` ([#4252](https://github.com/quiltdata/quilt/pull/4252)) - [Fixed] Resolve caching issues where changes in `.quilt/{workflows,catalog}` were not applied ([#4245](https://github.com/quiltdata/quilt/pull/4245)) diff --git a/catalog/app/containers/Admin/Settings/Settings.tsx b/catalog/app/containers/Admin/Settings/Settings.tsx index 0dc05e0265d..17445a3ea6d 100644 --- a/catalog/app/containers/Admin/Settings/Settings.tsx +++ b/catalog/app/containers/Admin/Settings/Settings.tsx @@ -12,6 +12,7 @@ import * as validators from 'utils/validators' import * as Form from '../Form' import SearchSettings from './SearchSettings' +import TabulatorSettings from './TabulatorSettings' import ThemeEditor from './ThemeEditor' function useBeta(): [boolean, (b: boolean) => Promise] { @@ -273,6 +274,10 @@ const useStyles = M.makeStyles((t) => ({ title: { margin: t.spacing(0, 0, 2), padding: t.spacing(0, 2), + + '* + &': { + marginTop: t.spacing(2), + }, }, })) @@ -324,6 +329,13 @@ export default function Settings() { + + + Tabulator Settings + + + + ) } diff --git a/catalog/app/containers/Admin/Settings/TabulatorSettings.tsx b/catalog/app/containers/Admin/Settings/TabulatorSettings.tsx new file mode 100644 index 00000000000..ab406002ace --- /dev/null +++ b/catalog/app/containers/Admin/Settings/TabulatorSettings.tsx @@ -0,0 +1,90 @@ +import * as React from 'react' +import * as M from '@material-ui/core' +import * as Lab from '@material-ui/lab' +import * as Sentry from '@sentry/react' + +import Skeleton from 'components/Skeleton' +import { docs } from 'constants/urls' +import * as Notifications from 'containers/Notifications' +import * as GQL from 'utils/GraphQL' +import StyledLink from 'utils/StyledLink' + +import OPEN_QUERY_QUERY from './gql/TabulatorOpenQuery.generated' +import SET_OPEN_QUERY_MUTATION from './gql/SetTabulatorOpenQuery.generated' + +interface ToggleProps { + checked: boolean +} + +function Toggle({ checked }: ToggleProps) { + const { push: notify } = Notifications.use() + const mutate = GQL.useMutation(SET_OPEN_QUERY_MUTATION) + const [mutation, setMutation] = React.useState<{ enabled: boolean } | null>(null) + + const handleChange = React.useCallback( + async (_event, enabled: boolean) => { + if (mutation) return + setMutation({ enabled }) + try { + await mutate({ enabled }) + } catch (e) { + Sentry.captureException(e) + notify(`Failed to update tabulator settings: ${e}`) + } finally { + setMutation(null) + } + }, + [mutate, notify, mutation], + ) + + return ( + <> + + } + label="Enable open querying of Tabulator tables" + /> + + CAUTION: When enabled, Tabulator defers all access control to AWS and does + not enforce any extra restrictions.{' '} + + Learn more + {' '} + in the documentation. + + + ) +} + +export default function TabulatorSettings() { + const query = GQL.useQuery(OPEN_QUERY_QUERY) + + return ( + + {GQL.fold(query, { + data: ({ admin }) => , + fetching: () => ( + <> + + + + ), + error: (e) => ( + + Could not fetch tabulator settings: +
+ {e.message} + + ), + })} + + ) +} diff --git a/catalog/app/containers/Admin/Settings/gql/SetTabulatorOpenQuery.generated.ts b/catalog/app/containers/Admin/Settings/gql/SetTabulatorOpenQuery.generated.ts new file mode 100644 index 00000000000..6f68ab6cc75 --- /dev/null +++ b/catalog/app/containers/Admin/Settings/gql/SetTabulatorOpenQuery.generated.ts @@ -0,0 +1,84 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core' +import * as Types from '../../../../model/graphql/types.generated' + +export type containers_Admin_Settings_gql_SetTabulatorOpenQueryMutationVariables = + Types.Exact<{ + enabled: Types.Scalars['Boolean'] + }> + +export type containers_Admin_Settings_gql_SetTabulatorOpenQueryMutation = { + readonly __typename: 'Mutation' +} & { + readonly admin: { readonly __typename: 'AdminMutations' } & { + readonly setTabulatorOpenQuery: { + readonly __typename: 'TabulatorOpenQueryResult' + } & Pick + } +} + +export const containers_Admin_Settings_gql_SetTabulatorOpenQueryDocument = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'mutation', + name: { + kind: 'Name', + value: 'containers_Admin_Settings_gql_SetTabulatorOpenQuery', + }, + variableDefinitions: [ + { + kind: 'VariableDefinition', + variable: { kind: 'Variable', name: { kind: 'Name', value: 'enabled' } }, + type: { + kind: 'NonNullType', + type: { kind: 'NamedType', name: { kind: 'Name', value: 'Boolean' } }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'admin' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'setTabulatorOpenQuery' }, + arguments: [ + { + kind: 'Argument', + name: { kind: 'Name', value: 'enabled' }, + value: { + kind: 'Variable', + name: { kind: 'Name', value: 'enabled' }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'tabulatorOpenQuery' }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode< + containers_Admin_Settings_gql_SetTabulatorOpenQueryMutation, + containers_Admin_Settings_gql_SetTabulatorOpenQueryMutationVariables +> + +export { containers_Admin_Settings_gql_SetTabulatorOpenQueryDocument as default } diff --git a/catalog/app/containers/Admin/Settings/gql/SetTabulatorOpenQuery.graphql b/catalog/app/containers/Admin/Settings/gql/SetTabulatorOpenQuery.graphql new file mode 100644 index 00000000000..8a7ead43a38 --- /dev/null +++ b/catalog/app/containers/Admin/Settings/gql/SetTabulatorOpenQuery.graphql @@ -0,0 +1,7 @@ +mutation($enabled: Boolean!) { + admin { + setTabulatorOpenQuery(enabled: $enabled) { + tabulatorOpenQuery + } + } +} diff --git a/catalog/app/containers/Admin/Settings/gql/TabulatorOpenQuery.generated.ts b/catalog/app/containers/Admin/Settings/gql/TabulatorOpenQuery.generated.ts new file mode 100644 index 00000000000..8a68745e323 --- /dev/null +++ b/catalog/app/containers/Admin/Settings/gql/TabulatorOpenQuery.generated.ts @@ -0,0 +1,47 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core' +import * as Types from '../../../../model/graphql/types.generated' + +export type containers_Admin_Settings_gql_TabulatorOpenQueryQueryVariables = Types.Exact<{ + [key: string]: never +}> + +export type containers_Admin_Settings_gql_TabulatorOpenQueryQuery = { + readonly __typename: 'Query' +} & { + readonly admin: { readonly __typename: 'AdminQueries' } & Pick< + Types.AdminQueries, + 'tabulatorOpenQuery' + > +} + +export const containers_Admin_Settings_gql_TabulatorOpenQueryDocument = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'query', + name: { kind: 'Name', value: 'containers_Admin_Settings_gql_TabulatorOpenQuery' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'admin' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'tabulatorOpenQuery' } }, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode< + containers_Admin_Settings_gql_TabulatorOpenQueryQuery, + containers_Admin_Settings_gql_TabulatorOpenQueryQueryVariables +> + +export { containers_Admin_Settings_gql_TabulatorOpenQueryDocument as default } diff --git a/catalog/app/containers/Admin/Settings/gql/TabulatorOpenQuery.graphql b/catalog/app/containers/Admin/Settings/gql/TabulatorOpenQuery.graphql new file mode 100644 index 00000000000..dd7fd5a7762 --- /dev/null +++ b/catalog/app/containers/Admin/Settings/gql/TabulatorOpenQuery.graphql @@ -0,0 +1,5 @@ +query { + admin { + tabulatorOpenQuery + } +} diff --git a/catalog/app/model/graphql/schema.generated.ts b/catalog/app/model/graphql/schema.generated.ts index be04791e97a..20ad97f7ab2 100644 --- a/catalog/app/model/graphql/schema.generated.ts +++ b/catalog/app/model/graphql/schema.generated.ts @@ -240,9 +240,37 @@ export default { }, ], }, + { + name: 'setTabulatorOpenQuery', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'OBJECT', + name: 'TabulatorOpenQueryResult', + ofType: null, + }, + }, + args: [ + { + name: 'enabled', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'Boolean', + ofType: null, + }, + }, + }, + ], + }, ], interfaces: [], }, + { + kind: 'SCALAR', + name: 'Boolean', + }, { kind: 'OBJECT', name: 'AdminQueries', @@ -280,13 +308,21 @@ export default { }, args: [], }, + { + name: 'tabulatorOpenQuery', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'Boolean', + ofType: null, + }, + }, + args: [], + }, ], interfaces: [], }, - { - kind: 'SCALAR', - name: 'Boolean', - }, { kind: 'OBJECT', name: 'BooleanPackageUserMetaFacet', @@ -5437,6 +5473,25 @@ export default { }, ], }, + { + kind: 'OBJECT', + name: 'TabulatorOpenQueryResult', + fields: [ + { + name: 'tabulatorOpenQuery', + type: { + kind: 'NON_NULL', + ofType: { + kind: 'SCALAR', + name: 'Boolean', + ofType: null, + }, + }, + args: [], + }, + ], + interfaces: [], + }, { kind: 'OBJECT', name: 'TabulatorTable', diff --git a/catalog/app/model/graphql/types.generated.ts b/catalog/app/model/graphql/types.generated.ts index fb5d1b2a862..8b401788ada 100644 --- a/catalog/app/model/graphql/types.generated.ts +++ b/catalog/app/model/graphql/types.generated.ts @@ -48,6 +48,7 @@ export interface AdminMutations { readonly setSsoConfig: Maybe readonly bucketSetTabulatorTable: BucketSetTabulatorTableResult readonly bucketRenameTabulatorTable: BucketSetTabulatorTableResult + readonly setTabulatorOpenQuery: TabulatorOpenQueryResult } export interface AdminMutationssetSsoConfigArgs { @@ -66,11 +67,16 @@ export interface AdminMutationsbucketRenameTabulatorTableArgs { newTableName: Scalars['String'] } +export interface AdminMutationssetTabulatorOpenQueryArgs { + enabled: Scalars['Boolean'] +} + export interface AdminQueries { readonly __typename: 'AdminQueries' readonly user: UserAdminQueries readonly ssoConfig: Maybe readonly isDefaultRoleSettingDisabled: Scalars['Boolean'] + readonly tabulatorOpenQuery: Scalars['Boolean'] } export interface BooleanPackageUserMetaFacet extends IPackageUserMetaFacet { @@ -1167,6 +1173,11 @@ export interface SubscriptionState { export type SwitchRoleResult = Me | InvalidInput | OperationError +export interface TabulatorOpenQueryResult { + readonly __typename: 'TabulatorOpenQueryResult' + readonly tabulatorOpenQuery: Scalars['Boolean'] +} + export interface TabulatorTable { readonly __typename: 'TabulatorTable' readonly name: Scalars['String'] diff --git a/catalog/app/utils/GraphQL/Provider.tsx b/catalog/app/utils/GraphQL/Provider.tsx index 05c34cd7238..87a36c05b01 100644 --- a/catalog/app/utils/GraphQL/Provider.tsx +++ b/catalog/app/utils/GraphQL/Provider.tsx @@ -347,6 +347,18 @@ export default function GraphQLProvider({ children }: React.PropsWithChildren<{} cache.invalidate({ __typename: 'Query' }, 'admin') cache.invalidate({ __typename: 'Query' }, 'roles') } + if (result.admin?.setTabulatorOpenQuery?.tabulatorOpenQuery != null) { + cache.updateQuery( + { query: urql.gql`{ admin { tabulatorOpenQuery } }` }, + ({ admin }) => ({ + admin: { + ...admin, + tabulatorOpenQuery: + result.admin.setTabulatorOpenQuery.tabulatorOpenQuery, + }, + }), + ) + } }, }, }, diff --git a/docs/advanced-features/tabulator.md b/docs/advanced-features/tabulator.md index fcc34188407..d8c28f26119 100644 --- a/docs/advanced-features/tabulator.md +++ b/docs/advanced-features/tabulator.md @@ -49,10 +49,10 @@ parser: column names in the document. For Parquet, they must match except for case. However, if column names are present in a CSV/TSV, you must set `header` to `true` in the parser configuration. -1. **Types**: Must be uppercase and match the [Apache Arrow Data -Types](https://docs.aws.amazon.com/athena/latest/ug/data-types.html) -used by Amazon Athena. Valid types are BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, -FLOAT, DOUBLE, STRING, BINARY, DATE, TIMESTAMP. +1. **Types**: Must be uppercase and match the + [Apache Arrow Data Types](https://docs.aws.amazon.com/athena/latest/ug/data-types.html) + used by Amazon Athena. Valid types are BOOLEAN, TINYINT, SMALLINT, INT, + BIGINT, FLOAT, DOUBLE, STRING, BINARY, DATE, TIMESTAMP. 1. **Source**: The source defines the packages and objects to query. The `type` must be `quilt-packages`. The `package_name` is a regular expression that matches the package names to include. The `logical_key` is a regular @@ -78,11 +78,15 @@ In addition to the columns defined in the schema, Tabulator will add: ### Using Athena to Access Tabulator -Due to the way permissions are configured, Tabulator cannot be accessed from the -AWS Console or Athena views. You must access Tabulator via the Quilt stack in -order to query those tables. This can be done by users via the per-bucket -"Queries" tab in the Quilt Catalog, or programmatically via `quilt3`. See -"Usage" below for more details. +The primary way of accessing Tabulator is using the Quilt stack to query those +tables. This can be done by users via the per-bucket "Queries" tab in the Quilt +Catalog, or programmatically via `quilt3`. See "Usage" below for more details. + +As of Quilt Platform version 1.57, admins can enable [open query](#open-query) +(below) to allow external users to access Tabulator tables directly from the AWS +Console, Athena views, or JDBC connectors. This is especially useful for +customers who want to access Tabulator from external services, such as Tableau +and Spotfire. ### Caveats @@ -110,7 +114,8 @@ order to query those tables. This can be done by users via the per-bucket Once the configuration is set, users can query the tables using the Athena tab from the Quilt Catalog. Note that because Tabulator runs with elevated -permissions, it cannot be accessed from the AWS Console. +permissions, it cannot be accessed from the AWS Console by default +(unless [open query](#open-query) is enabled). For example, to query the `ccle_tsv` table from the appropriate workgroup in the `quilt-tf-stable` stack, where the database (bucket name) is `udp-spec`: @@ -128,7 +133,7 @@ SELECT * FROM "userathenadatabase-1qstaay0czbf"."udp-spec_packages-view" LIMIT 10 ``` -We can then join on PKG_NAME to add the `user_meta` field from the package +We can then join on `PKG_NAME` to add the `user_meta` field from the package metadata to the tabulated results: ```sql @@ -148,6 +153,9 @@ page from which you must paste in the appropriate access token. Use `get_boto3_session()` to get a session with the same permissions as your Quilt Catalog user, then use the `boto3` Athena client to run queries. +> If [open query](#open-query) is enabled, you can use any +> AWS credentials providing access to Athena resources associated with Tabulator. + Here is a complete example: @@ -188,3 +196,37 @@ if state == 'SUCCEEDED': else: print(f'Query did not succeed. Final state: {state}') ``` + +## Open Query + +> Available since Quilt Platform version 1.57 + +By default, Tabulator is only accessible via a session provided by the Quilt +Catalog, and the access is scoped to the permissions of the Catalog user +associated with that session. However, admins can choose to enable **open +query** to Tabulator tables, deferring all access control to AWS, thus enabling +access from external services. This allows querying Tabulator from the AWS +Console, Athena views or JDBC connectors -- as long as the caller has been +granted the necessary permissions to access Athena resources associated with +Tabulator. + +### 1. Enable Open Query + +An admin can enable open query via the Admin UI: + +![Tabulator Settings](../imgs/admin-tabulator-settings.png) + +### 2. Configure Permissions + +In order to access Tabulator in open query mode, the caller must use a special +workgroup, and have permissions to use that workgroup and access tabulator +resources. For convenience, Quilt Stack provides a pre-configured workgroup and +policy for open query -- they can be found in the stack outputs: + +1. `TabulatorOpenQueryPolicyArn`: attach this managed policy to a relevant IAM + role (or copy the statements directly to your own role/policy). + +2. `TabulatorOpenQueryWorkGroup`: configure your Athena client or connector to + use this workgroup (or create your own with the same results output configuration). + +![Tabulator Resources](../imgs/admin-tabulator-resources.png) diff --git a/docs/imgs/admin-tabulator-resources.png b/docs/imgs/admin-tabulator-resources.png new file mode 100644 index 00000000000..d8f20c3c36c Binary files /dev/null and b/docs/imgs/admin-tabulator-resources.png differ diff --git a/docs/imgs/admin-tabulator-settings.png b/docs/imgs/admin-tabulator-settings.png new file mode 100644 index 00000000000..1710de372b6 Binary files /dev/null and b/docs/imgs/admin-tabulator-settings.png differ diff --git a/shared/graphql/schema.graphql b/shared/graphql/schema.graphql index ea342cd5806..1238025dd74 100644 --- a/shared/graphql/schema.graphql +++ b/shared/graphql/schema.graphql @@ -526,6 +526,7 @@ type AdminQueries { user: UserAdminQueries! ssoConfig: SsoConfig isDefaultRoleSettingDisabled: Boolean! + tabulatorOpenQuery: Boolean! } type MyRole { @@ -904,11 +905,16 @@ union SetSsoConfigResult = SsoConfig | InvalidInput | OperationError union BucketSetTabulatorTableResult = BucketConfig | InvalidInput | OperationError +type TabulatorOpenQueryResult { + tabulatorOpenQuery: Boolean! +} + type AdminMutations { user: UserAdminMutations! setSsoConfig(config: String): SetSsoConfigResult bucketSetTabulatorTable(bucketName: String!, tableName: String!, config: String): BucketSetTabulatorTableResult! bucketRenameTabulatorTable(bucketName: String!, tableName: String!, newTableName: String!): BucketSetTabulatorTableResult! + setTabulatorOpenQuery(enabled: Boolean!): TabulatorOpenQueryResult! } type Mutation {