diff --git a/docs/docusaurus/docs/cloud/data_assets/manage_data_assets.md b/docs/docusaurus/docs/cloud/data_assets/manage_data_assets.md index 40efa48a9221..a4230a66ea4f 100644 --- a/docs/docusaurus/docs/cloud/data_assets/manage_data_assets.md +++ b/docs/docusaurus/docs/cloud/data_assets/manage_data_assets.md @@ -303,29 +303,29 @@ You can only edit the settings of Data Assets created in GX Cloud. 3. Click **Save**. -## Secure your GX API Data Source connection strings +## Data Source credential management -When you use the GX API and not GX Cloud to connect to Data Sources, you can obfuscate sensitive Data Source credentials in your connection string as an additional security measure. +To connect to your Data Source in GX Cloud, there are two methods for managing credentials: -1. Store your credential value as an environment variable by entering `export ENV_VAR_NAME=env_var_value` in the terminal or adding the command to your `~/.bashrc` or `~/.zshrc` file. For example: +1. **Direct input**: You can input credentials directly into GX Cloud. These credentials are stored in GX Cloud and securely encrypted at rest and in transit. When Data Source credentials have been directly provided, they can be used to connect to a Data Source in any GX Cloud deployment pattern. - ```bash title="Terminal input" - export GX_CLOUD_SNOWFLAKE_PASSWORD= - ``` - Prefix environment variable names with `GX_CLOUD_`. +2. **Environment variable substitution**: To enhance security, you can use environment variables to manage sensitive connection parameters or strings. For example, instead of directly including your database password in configuration settings, you can use a variable reference like `${MY_DATABASE_PASSWORD}`. When using environment variable substitution, your password is not stored or transmitted to GX Cloud. -2. Create a Data Source connection string using the environment variable name instead of the credential value. For example: +:::warning[Environment variable substitution support] +Environment variable substitution is not supported in fully hosted deployments. +::: - ```python title="Example Data Source connection string" - snowflake://:${GX_CLOUD_SNOWFLAKE_PASSWORD}@//?warehouse=&role= - ``` - Environment variable names must be enclosed by curly braces and be preceded by a dollar sign. For example: `${GX_CLOUD_SNOWFLAKE_PASSWORD}`. Do not use interpolation to add credential values to connection strings. + - **Configure the environment variable**: Enter the name of your environment variable, enclosed in `${}`, into the Data Source setup form. For instance, you might use `${MY_DATABASE_PASSWORD}`. -3. Use the environment variable to supply the credential value when you run the GX Agent. For example: + - **Inject the variable into your GX Agent container or environment**: When running the GX Agent Docker container, include the environment variable in the command. For example: + + ```bash title="Terminal input" + docker run -it -e MY_DATABASE_PASSWORD= -e GX_CLOUD_ACCESS_TOKEN= -e GX_CLOUD_ORGANIZATION_ID= greatexpectations/agent:stable + ``` - ```bash title="Terminal input" - docker run --rm -e GX_CLOUD_SNOWFLAKE_PASSWORD="" -e GX_CLOUD_ACCESS_TOKEN="" -e GX_CLOUD_ORGANIZATION_ID="" greatexpectations/agent - ``` +When running the GX Agent in other Docker-based service, including Kubernetes, ECS, ACI, and GCE, use the service's instructions to set and provide environment variables to the running container. + +When using environment variable substitution in a read-only deployment, set the environment variable in the environment where the GX Core Python client is running. ## Delete a Data Asset diff --git a/docs/docusaurus/sidebars.js b/docs/docusaurus/sidebars.js index d503b215abe9..2d783bfd6ddb 100644 --- a/docs/docusaurus/sidebars.js +++ b/docs/docusaurus/sidebars.js @@ -3,7 +3,7 @@ module.exports = { { type: 'category', label: 'Introduction to GX Core', - link: {type: 'doc', id: 'core/introduction/introduction'}, + link: { type: 'doc', id: 'core/introduction/introduction' }, items: [ { type: 'doc', @@ -20,7 +20,7 @@ module.exports = { { type: 'category', label: 'Set up a GX environment', - link: {type: 'doc', id: 'core/set_up_a_gx_environment/set_up_a_gx_environment'}, + link: { type: 'doc', id: 'core/set_up_a_gx_environment/set_up_a_gx_environment' }, items: [ { type: 'doc', @@ -47,7 +47,7 @@ module.exports = { { type: 'category', label: 'Connect to data', - link: {type: 'doc', id: 'core/connect_to_data/connect_to_data'}, + link: { type: 'doc', id: 'core/connect_to_data/connect_to_data' }, items: [ { type: 'doc', @@ -96,42 +96,42 @@ module.exports = { { type: 'category', label: 'Run Validations', - link: {type: 'doc', id: 'core/run_validations/run_validations'}, + link: { type: 'doc', id: 'core/run_validations/run_validations' }, items: [ - {type: 'doc', id: 'core/run_validations/create_a_validation_definition'}, - {type: 'doc', id: 'core/run_validations/run_a_validation_definition'}, + { type: 'doc', id: 'core/run_validations/create_a_validation_definition' }, + { type: 'doc', id: 'core/run_validations/run_a_validation_definition' }, ] }, { type: 'category', label: 'Trigger actions based on results', - link: {type: 'doc', id: 'core/trigger_actions_based_on_results/trigger_actions_based_on_results'}, + link: { type: 'doc', id: 'core/trigger_actions_based_on_results/trigger_actions_based_on_results' }, items: [ - {type: 'doc', id: 'core/trigger_actions_based_on_results/create_a_checkpoint_with_actions'}, - {type: 'doc', id: 'core/trigger_actions_based_on_results/choose_a_result_format/choose_a_result_format'}, - {type: 'doc', id: 'core/trigger_actions_based_on_results/run_a_checkpoint'}, + { type: 'doc', id: 'core/trigger_actions_based_on_results/create_a_checkpoint_with_actions' }, + { type: 'doc', id: 'core/trigger_actions_based_on_results/choose_a_result_format/choose_a_result_format' }, + { type: 'doc', id: 'core/trigger_actions_based_on_results/run_a_checkpoint' }, ] }, { type: 'category', label: 'Customize Expectations', - link: {type: 'doc', id: 'core/customize_expectations/customize_expectations'}, + link: { type: 'doc', id: 'core/customize_expectations/customize_expectations' }, items: [ - {type: 'doc', id: 'core/customize_expectations/expectation_row_conditions'}, - {type: 'doc', id: 'core/customize_expectations/define_a_custom_expectation_class'}, - {type: 'doc', id: 'core/customize_expectations/use_sql_to_define_a_custom_expectation'}, + { type: 'doc', id: 'core/customize_expectations/expectation_row_conditions' }, + { type: 'doc', id: 'core/customize_expectations/define_a_custom_expectation_class' }, + { type: 'doc', id: 'core/customize_expectations/use_sql_to_define_a_custom_expectation' }, ] }, { type: 'category', label: 'Configure project settings', - link: {type: 'doc', id: 'core/configure_project_settings/configure_project_settings'}, + link: { type: 'doc', id: 'core/configure_project_settings/configure_project_settings' }, items: [ - {type: 'doc', id: 'core/configure_project_settings/configure_metadata_stores/configure_metadata_stores'}, - {type: 'doc', id: 'core/configure_project_settings/configure_data_docs/configure_data_docs'}, - {type: 'doc', id: 'core/configure_project_settings/configure_credentials/configure_credentials'}, - {type: 'doc', id: 'core/configure_project_settings/access_secrets_managers/access_secrets_managers'}, - {type: 'doc', id: 'core/configure_project_settings/toggle_analytics_events/toggle_analytics_events'} + { type: 'doc', id: 'core/configure_project_settings/configure_metadata_stores/configure_metadata_stores' }, + { type: 'doc', id: 'core/configure_project_settings/configure_data_docs/configure_data_docs' }, + { type: 'doc', id: 'core/configure_project_settings/configure_credentials/configure_credentials' }, + { type: 'doc', id: 'core/configure_project_settings/access_secrets_managers/access_secrets_managers' }, + { type: 'doc', id: 'core/configure_project_settings/toggle_analytics_events/toggle_analytics_events' } ] }, { @@ -146,265 +146,265 @@ module.exports = { } ], gx_cloud: [ + { + type: 'category', + label: 'GX Cloud overview', + link: { type: 'doc', id: 'cloud/overview/gx_cloud_overview' }, + items: [ { - type: 'category', - label: 'GX Cloud overview', - link: { type: 'doc', id: 'cloud/overview/gx_cloud_overview' }, - items: [ - { - type: 'link', - label: 'GX Cloud concepts', - href: '/docs/cloud/overview/gx_cloud_overview#gx-cloud-concepts', - }, - { - type: 'link', - label: 'GX Cloud workflow', - href: '/docs/cloud/overview/gx_cloud_overview#gx-cloud-workflow', - }, - { - type: 'link', - label: 'GX Cloud architecture', - href: '/docs/cloud/overview/gx_cloud_overview#gx-cloud-architecture', - }, - ] - }, - { - type: 'category', - label: 'Deploy GX Cloud', - link: { type: 'doc', id: 'cloud/deploy/deploy_lp' }, - items: [ - 'cloud/deploy/deployment_patterns', - 'cloud/deploy/deploy_gx_agent', - ] - }, - { - type: 'category', - label: 'Connect GX Cloud', - link: { type: 'doc', id: 'cloud/connect/connect_lp' }, - items: [ - 'cloud/connect/connect_postgresql', - 'cloud/connect/connect_snowflake', - 'cloud/connect/connect_databrickssql', - 'cloud/connect/connect_airflow', - 'cloud/connect/connect_python', - ] - }, - { - type: 'category', - label: 'Manage Data Assets', - link: { type: 'doc', id: 'cloud/data_assets/manage_data_assets' }, - items: [ - { - type: 'link', - label: 'Create a Data Asset', - href: '/docs/cloud/data_assets/manage_data_assets#create-a-data-asset', - }, - { - type: 'link', - label: 'View Data Asset metrics', - href: '/docs/cloud/data_assets/manage_data_assets#view-data-asset-metrics', - }, - { - type: 'link', - label: 'Add an Expectation to a Data Asset column', - href: '/docs/cloud/data_assets/manage_data_assets#add-an-expectation-to-a-data-asset-column', - }, - { - type: 'link', - label: 'Add a Data Asset to an Existing Data Source', - href: '/docs/cloud/data_assets/manage_data_assets#add-a-data-asset-to-an-existing-data-source', - }, - { - type: 'link', - label: 'Edit Data Source settings', - href: '/docs/cloud/data_assets/manage_data_assets#edit-data-source-settings', - }, - { - type: 'link', - label: 'Edit a Data Asset', - href: '/docs/cloud/data_assets/manage_data_assets#edit-a-data-asset', - }, - { - type: 'link', - label: 'Secure your GX API Data Source connection strings', - href: '/docs/cloud/data_assets/manage_data_assets#secure-your-gx-api-data-source-connection-strings', - }, - { - type: 'link', - label: 'Delete a Data Asset', - href: '/docs/cloud/data_assets/manage_data_assets#delete-a-data-asset', - }, - ] - }, - { - type: 'category', - label: 'Manage Expectations', - link: { type: 'doc', id: 'cloud/expectations/manage_expectations' }, - items: [ - { - type: 'link', - label: 'Available Expectations', - href: '/docs/cloud/expectations/manage_expectations#available-expectations', - }, - { - type: 'link', - label: 'Custom SQL Expectations', - href: '/docs/cloud/expectations/manage_expectations#custom-sql-expectations', - }, - { - type: 'link', - label: 'Add an Expectation', - href: '/docs/cloud/expectations/manage_expectations#add-an-expectation', - }, - { - type: 'link', - label: 'Edit an Expectation', - href: '/docs/cloud/expectations/manage_expectations#edit-an-expectation', - }, - { - type: 'link', - label: 'View Expectation history', - href: '/docs/cloud/expectations/manage_expectations#view-expectation-history', - }, - { - type: 'link', - label: 'Delete an Expectation', - href: '/docs/cloud/expectations/manage_expectations#delete-an-expectation', - }, - ] - }, - { - type: 'category', - label: 'Manage Expectation Suites', - link: { type: 'doc', id: 'cloud/expectation_suites/manage_expectation_suites' }, - items: [ - { - type: 'link', - label: 'Create an Expectation Suite ', - href: '/docs/cloud/expectation_suites/manage_expectation_suites#create-an-expectation-suite', - }, - { - type: 'link', - label: 'Edit an Expectation Suite name', - href: '/docs/cloud/expectation_suites/manage_expectation_suites#edit-an-expectation-suite-name', - }, - { - type: 'link', - label: 'Delete an Expectation Suite', - href: '/docs/cloud/expectation_suites/manage_expectation_suites#delete-an-expectation-suite', - }, - ] - }, - { - type: 'category', - label: 'Manage Validations', - link: { type: 'doc', id: 'cloud/validations/manage_validations' }, - items: [ - { - type: 'link', - label: 'Run a Validation', - href: '/docs/cloud/validations/manage_validations#run-a-validation', - }, - { - type: 'link', - label: 'Run a Validation on a Data Asset containing partitions', - href: '/docs/cloud/validations/manage_validations#run-a-validation-on-a-data-asset-containing-partitions', - }, - { - type: 'link', - label: 'View Validation run history', - href: '/docs/cloud/validations/manage_validations#view-validation-run-history', - }, - ] - }, - { - type: 'category', - label: 'Manage schedules', - link: { type: 'doc', id: 'cloud/schedules/manage_schedules' }, - items: [ - { - type: 'link', - label: 'Add a schedule', - href: '/docs/cloud/schedules/manage_schedules#create-a-schedule-for-an-existing-expectation-suite', - }, - { - type: 'link', - label: 'Edit a schedule', - href: '/docs/cloud/schedules/manage_schedules#edit-a-schedule', - }, - { - type: 'link', - label: 'Disable a schedule', - href: '/docs/cloud/schedules/manage_schedules#disable-a-schedule', - }, - ] - }, - { - type: 'category', - label: 'Manage alerts', - link: { type: 'doc', id: 'cloud/alerts/manage_alerts' }, - items: [ - { - type: "link", - label: "Email alert default settings", - href: "/docs/cloud/alerts/manage_alerts#email-alert-default-settings" - }, - { - type: "link", - label: "Update an email alert", - href: "/docs/cloud/alerts/manage_alerts#update-an-email-alert" - }, - ] - }, - { - type: 'category', - label: 'Manage users and access tokens', - link: { type: 'doc', id: 'cloud/users/manage_users' }, - items: [ - { - type: 'link', - label: 'Roles and responsibilities', - href: '/docs/cloud/users/manage_users#roles-and-responsibilities', - }, - { - type: 'link', - label: 'Invite a user', - href: '/docs/cloud/users/manage_users#invite-a-user', - }, - { - type: 'link', - label: 'Edit a user role', - href: '/docs/cloud/users/manage_users#edit-a-user-role', - }, - { - type: 'link', - label: 'Delete a user', - href: '/docs/cloud/users/manage_users#delete-a-user', - }, - { - type: 'link', - label: 'Create a user access token', - href: '/docs/cloud/users/manage_users#create-a-user-access-token', - }, - { - type: 'link', - label: 'Create an organization access token', - href: '/docs/cloud/users/manage_users#create-an-organization-access-token', - }, - { - type: 'link', - label: 'Delete a user or organization access token', - href: '/docs/cloud/users/manage_users#delete-a-user-or-organization-access-token', - }, - ] - }, - { - type:'link', - label: 'Request a demo for GX Cloud', - href: 'https://www.greatexpectations.io/demo', - className: 'request-demo-sidebar', + type: 'link', + label: 'GX Cloud concepts', + href: '/docs/cloud/overview/gx_cloud_overview#gx-cloud-concepts', }, - ], + { + type: 'link', + label: 'GX Cloud workflow', + href: '/docs/cloud/overview/gx_cloud_overview#gx-cloud-workflow', + }, + { + type: 'link', + label: 'GX Cloud architecture', + href: '/docs/cloud/overview/gx_cloud_overview#gx-cloud-architecture', + }, + ] + }, + { + type: 'category', + label: 'Deploy GX Cloud', + link: { type: 'doc', id: 'cloud/deploy/deploy_lp' }, + items: [ + 'cloud/deploy/deployment_patterns', + 'cloud/deploy/deploy_gx_agent', + ] + }, + { + type: 'category', + label: 'Connect GX Cloud', + link: { type: 'doc', id: 'cloud/connect/connect_lp' }, + items: [ + 'cloud/connect/connect_postgresql', + 'cloud/connect/connect_snowflake', + 'cloud/connect/connect_databrickssql', + 'cloud/connect/connect_airflow', + 'cloud/connect/connect_python', + ] + }, + { + type: 'category', + label: 'Manage Data Assets', + link: { type: 'doc', id: 'cloud/data_assets/manage_data_assets' }, + items: [ + { + type: 'link', + label: 'Create a Data Asset', + href: '/docs/cloud/data_assets/manage_data_assets#create-a-data-asset', + }, + { + type: 'link', + label: 'View Data Asset metrics', + href: '/docs/cloud/data_assets/manage_data_assets#view-data-asset-metrics', + }, + { + type: 'link', + label: 'Add an Expectation to a Data Asset column', + href: '/docs/cloud/data_assets/manage_data_assets#add-an-expectation-to-a-data-asset-column', + }, + { + type: 'link', + label: 'Add a Data Asset to an Existing Data Source', + href: '/docs/cloud/data_assets/manage_data_assets#add-a-data-asset-to-an-existing-data-source', + }, + { + type: 'link', + label: 'Edit Data Source settings', + href: '/docs/cloud/data_assets/manage_data_assets#edit-data-source-settings', + }, + { + type: 'link', + label: 'Edit a Data Asset', + href: '/docs/cloud/data_assets/manage_data_assets#edit-a-data-asset', + }, + { + type: 'link', + label: 'Data Source credential management', + href: '/docs/cloud/data_assets/manage_data_assets#data-source-credential-management', + }, + { + type: 'link', + label: 'Delete a Data Asset', + href: '/docs/cloud/data_assets/manage_data_assets#delete-a-data-asset', + }, + ] + }, + { + type: 'category', + label: 'Manage Expectations', + link: { type: 'doc', id: 'cloud/expectations/manage_expectations' }, + items: [ + { + type: 'link', + label: 'Available Expectations', + href: '/docs/cloud/expectations/manage_expectations#available-expectations', + }, + { + type: 'link', + label: 'Custom SQL Expectations', + href: '/docs/cloud/expectations/manage_expectations#custom-sql-expectations', + }, + { + type: 'link', + label: 'Add an Expectation', + href: '/docs/cloud/expectations/manage_expectations#add-an-expectation', + }, + { + type: 'link', + label: 'Edit an Expectation', + href: '/docs/cloud/expectations/manage_expectations#edit-an-expectation', + }, + { + type: 'link', + label: 'View Expectation history', + href: '/docs/cloud/expectations/manage_expectations#view-expectation-history', + }, + { + type: 'link', + label: 'Delete an Expectation', + href: '/docs/cloud/expectations/manage_expectations#delete-an-expectation', + }, + ] + }, + { + type: 'category', + label: 'Manage Expectation Suites', + link: { type: 'doc', id: 'cloud/expectation_suites/manage_expectation_suites' }, + items: [ + { + type: 'link', + label: 'Create an Expectation Suite ', + href: '/docs/cloud/expectation_suites/manage_expectation_suites#create-an-expectation-suite', + }, + { + type: 'link', + label: 'Edit an Expectation Suite name', + href: '/docs/cloud/expectation_suites/manage_expectation_suites#edit-an-expectation-suite-name', + }, + { + type: 'link', + label: 'Delete an Expectation Suite', + href: '/docs/cloud/expectation_suites/manage_expectation_suites#delete-an-expectation-suite', + }, + ] + }, + { + type: 'category', + label: 'Manage Validations', + link: { type: 'doc', id: 'cloud/validations/manage_validations' }, + items: [ + { + type: 'link', + label: 'Run a Validation', + href: '/docs/cloud/validations/manage_validations#run-a-validation', + }, + { + type: 'link', + label: 'Run a Validation on a Data Asset containing partitions', + href: '/docs/cloud/validations/manage_validations#run-a-validation-on-a-data-asset-containing-partitions', + }, + { + type: 'link', + label: 'View Validation run history', + href: '/docs/cloud/validations/manage_validations#view-validation-run-history', + }, + ] + }, + { + type: 'category', + label: 'Manage schedules', + link: { type: 'doc', id: 'cloud/schedules/manage_schedules' }, + items: [ + { + type: 'link', + label: 'Add a schedule', + href: '/docs/cloud/schedules/manage_schedules#create-a-schedule-for-an-existing-expectation-suite', + }, + { + type: 'link', + label: 'Edit a schedule', + href: '/docs/cloud/schedules/manage_schedules#edit-a-schedule', + }, + { + type: 'link', + label: 'Disable a schedule', + href: '/docs/cloud/schedules/manage_schedules#disable-a-schedule', + }, + ] + }, + { + type: 'category', + label: 'Manage alerts', + link: { type: 'doc', id: 'cloud/alerts/manage_alerts' }, + items: [ + { + type: "link", + label: "Email alert default settings", + href: "/docs/cloud/alerts/manage_alerts#email-alert-default-settings" + }, + { + type: "link", + label: "Update an email alert", + href: "/docs/cloud/alerts/manage_alerts#update-an-email-alert" + }, + ] + }, + { + type: 'category', + label: 'Manage users and access tokens', + link: { type: 'doc', id: 'cloud/users/manage_users' }, + items: [ + { + type: 'link', + label: 'Roles and responsibilities', + href: '/docs/cloud/users/manage_users#roles-and-responsibilities', + }, + { + type: 'link', + label: 'Invite a user', + href: '/docs/cloud/users/manage_users#invite-a-user', + }, + { + type: 'link', + label: 'Edit a user role', + href: '/docs/cloud/users/manage_users#edit-a-user-role', + }, + { + type: 'link', + label: 'Delete a user', + href: '/docs/cloud/users/manage_users#delete-a-user', + }, + { + type: 'link', + label: 'Create a user access token', + href: '/docs/cloud/users/manage_users#create-a-user-access-token', + }, + { + type: 'link', + label: 'Create an organization access token', + href: '/docs/cloud/users/manage_users#create-an-organization-access-token', + }, + { + type: 'link', + label: 'Delete a user or organization access token', + href: '/docs/cloud/users/manage_users#delete-a-user-or-organization-access-token', + }, + ] + }, + { + type: 'link', + label: 'Request a demo for GX Cloud', + href: 'https://www.greatexpectations.io/demo', + className: 'request-demo-sidebar', + }, + ], gx_apis: [ { type: 'category', @@ -434,8 +434,8 @@ module.exports = { 'reference/learn/data_quality_use_cases/freshness' ] }, - 'reference/learn/usage_statistics', - 'reference/learn/glossary' + 'reference/learn/usage_statistics', + 'reference/learn/glossary' ], }