From 0fd5529206f722e640bc1ef3a8df859f3456d783 Mon Sep 17 00:00:00 2001 From: Rachel-Reverie <94694058+Rachel-Reverie@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:31:59 -0500 Subject: [PATCH] [DOCS] Puts 1.0 documentation example code for how to run validations under test (#10230) --- .../docs/components/examples_under_test.py | 26 ++++++ .../create_a_validation_definition.py | 69 ++++++++++++++++ .../_examples/run_a_validation_definition.py | 73 +++++++++++++++++ .../create_a_validation_definition.md | 81 +++---------------- .../run_a_validation_definition.md | 34 ++------ docs/docusaurus/docusaurus.config.js | 12 +++ docs/docusaurus/src/css/custom.scss | 4 + 7 files changed, 200 insertions(+), 99 deletions(-) diff --git a/docs/docusaurus/docs/components/examples_under_test.py b/docs/docusaurus/docs/components/examples_under_test.py index c5e1671684a9..27c911ffbd80 100644 --- a/docs/docusaurus/docs/components/examples_under_test.py +++ b/docs/docusaurus/docs/components/examples_under_test.py @@ -8,6 +8,29 @@ docs_tests = [] +docs_example_scripts_run_validations = [ + # Create a Validation Definition + IntegrationTestFixture( + # To test, run: + # pytest --docs-tests -k "docs_example_create_a_validation_definition" tests/integration/test_script_runner.py + name="docs_example_create_a_validation_definition", + user_flow_script="docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py", + data_dir="docs/docusaurus/docs/components/_testing/test_data_sets/single_test_file", + # data_context_dir="", + backend_dependencies=[], + ), + # Run a Validation Definition + IntegrationTestFixture( + # To test, run: + # pytest --docs-tests -k "docs_example_run_a_validation_definition" tests/integration/test_script_runner.py + name="docs_example_run_a_validation_definition", + user_flow_script="docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py", + data_dir="docs/docusaurus/docs/components/_testing/test_data_sets/single_test_file", + # data_context_dir="", + backend_dependencies=[], + ), +] + connect_to_filesystem_data_create_a_data_source = [ # Local, pandas/spark IntegrationTestFixture( @@ -250,6 +273,9 @@ # Extend the docs_tests list with the above sublists (only the docs_tests list is imported # into `test_script_runner.py` and actually used in CI checks). + +docs_tests.extend(docs_example_scripts_run_validations) + docs_tests.extend(connect_to_filesystem_data_create_a_data_source) docs_tests.extend(connect_to_filesystem_data_create_a_data_asset) docs_tests.extend(connect_to_filesystem_data_create_a_batch_definition) diff --git a/docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py b/docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py index e69de29bb2d1..3c83a096cef9 100644 --- a/docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py +++ b/docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py @@ -0,0 +1,69 @@ +""" +This is an example script for how to create a Validation Definition. + +To test, run: +pytest --docs-tests -k "docs_example_create_a_validation_definition" tests/integration/test_script_runner.py +""" + + +def set_up_context_for_example(context): + # Create a Data Source + data_source = context.data_sources.add_pandas_filesystem( + name="my_data_source", base_directory="./data/folder_with_data" + ) + # Create a Data Asset + data_asset = data_source.add_csv_asset(name="my_data_asset") + # Create a Batch Definition + data_asset.add_batch_definition_path( + name="my_batch_definition", path="yellow_tripdata_sample_2019-01.csv" + ) + # Create an Expectation Suite + suite = context.suites.add(gx.ExpectationSuite(name="my_expectation_suite")) + # Add some Expectations + suite.add_expectation( + gx.expectations.ExpectColumnValuesToNotBeNull(column="pickup_datetime") + ) + suite.add_expectation( + gx.expectations.ExpectColumnValuesToNotBeNull(column="passenger_count") + ) + + +# EXAMPLE SCRIPT STARTS HERE: +# +import great_expectations as gx + +context = gx.get_context() +# Hide this +set_up_context_for_example(context) + +# Retrieve an Expectation Suite +# +expectation_suite_name = "my_expectation_suite" +expectation_suite = context.suites.get(name=expectation_suite_name) +# + +# Retrieve a Batch Definition +# +data_source_name = "my_data_source" +data_asset_name = "my_data_asset" +batch_definition_name = "my_batch_definition" +batch_definition = ( + context.get_datasource(data_source_name) + .get_asset(data_asset_name) + .get_batch_definition(batch_definition_name) +) +# + +# Create a Validation Definition +# +definition_name = "my_validation_definition" +validation_definition = gx.ValidationDefinition( + data=batch_definition, suite=expectation_suite, name=definition_name +) +# + +# Add the Validation Definition to the Data Context +# +validation_definition = context.validation_definitions.add(validation_definition) +# +# diff --git a/docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py b/docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py index e69de29bb2d1..a650a8485bd0 100644 --- a/docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py +++ b/docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py @@ -0,0 +1,73 @@ +""" +This is an example script for how to run a Validation Definition. + +To test, run: + +""" + + +def set_up_context_for_example(context): + # Create a Batch Definition + batch_definition = ( + context.data_sources.add_pandas_filesystem( + name="my_data_source", base_directory="./data/folder_with_data" + ) + .add_csv_asset(name="my_data_asset") + .add_batch_definition_path( + name="my_batch_definition", path="yellow_tripdata_sample_2019-01.csv" + ) + ) + + # Create an Expectation Suite + expectation_suite = context.suites.add( + gx.ExpectationSuite(name="my_expectation_suite") + ) + # Add some Expectations + expectation_suite.add_expectation( + gx.expectations.ExpectColumnValuesToNotBeNull(column="pickup_datetime") + ) + expectation_suite.add_expectation( + gx.expectations.ExpectColumnValuesToNotBeNull(column="passenger_count") + ) + # Create a Validation Definition + context.validation_definitions.add( + gx.ValidationDefinition( + data=batch_definition, + suite=expectation_suite, + name="my_validation_definition", + ) + ) + + +# EXAMPLE SCRIPT STARTS HERE: +# +import great_expectations as gx + +context = gx.get_context() +# Hide this +set_up_context_for_example(context) + +# Retrieve the Validation Definition +# +validation_definition_name = "my_validation_definition" +validation_definition = context.validation_definitions.get(validation_definition_name) +# + +# Run the Validation Definition +# +validation_results = validation_definition.run() +# + +# Review the Validation Results +# +print(validation_results) +# +# + + +# TODO: Set up the example script to use a GX Cloud Data Context and then +# add this portion back in. +# # Get the URL for Validation Results in GX Cloud +# # +# print(validation_results.results_url) +# # diff --git a/docs/docusaurus/docs/core/run_validations/create_a_validation_definition.md b/docs/docusaurus/docs/core/run_validations/create_a_validation_definition.md index b5a7c9f54974..ea53cb66d7d8 100644 --- a/docs/docusaurus/docs/core/run_validations/create_a_validation_definition.md +++ b/docs/docusaurus/docs/core/run_validations/create_a_validation_definition.md @@ -20,7 +20,7 @@ A Validation Definition is a fixed reference that links a Batch of data to an Ex - . - . -- . +- . In this guide the variable `context` is assumed to contain your Data Context. - . - . @@ -28,98 +28,37 @@ A Validation Definition is a fixed reference that links a Batch of data to an Ex -1. Import the `ValidationDefinition` class from the {GxData.product_name} library: - - ```python title="Python" - from great_expectations.core import ValidationDefinition - ``` - -2. Request a Data Context: - - ```python title="Python" - import great_expectations as gx - - context = gx.get_context() - ``` - -3. Retrieve an Expectation Suite with Expectations. +1. Retrieve an Expectation Suite with Expectations. Update the value of `suite_name` in the following code with the name of your Expectation Suite. Then execute the code to retrieve that Expectation Suite: - ```python title="Python" - suite_name = "" - suite = context.get_expectation_suite(suite_name) + ```python title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py - retrieve an Expectation Suite" ``` -4. Retrieve the Batch Definition that describes the data to associate with the Expectation Suite. +2. Retrieve the Batch Definition that describes the data to associate with the Expectation Suite. Update the values of `data_source_name`, `data_asset_name`, and `batch_definition_name` in the following code with the names of your previously defined Data Source, one of its Data Assets, and a Batch Definition for that Data Asset. Then execute the code to retrieve the Batch Definition: - ```python title="Python" - data_source_name = "my_datasource" - data_asset_name = "my_data_asset" - batch_definition_name = "my_batch_definition" - - batch_definition = context.get_datasource(data_source_name).get_asset(data_asset_name).get_batch_definition(batch_definition_name) + ```python title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py - retrieve a Batch Definition" ``` -5. Create a `ValidationDefinition` instance using the Batch Definition, Expectation Suite, and a unique name. +3. Create a `ValidationDefinition` instance using the Batch Definition, Expectation Suite, and a unique name. Update the value of `definition_name` with a descriptive name that indicates the purpose of the Validation Definition. Then execute the code to create your Validation Definition: - ```python title="Python" - definition_name = "My Validation Definition" - validation_definition = ValidationDefinition(data=batch_definition, suite=suite, name=definition_name) - ``` - -6. Optional. Save the Validation Definition to your Data Context. - - ```python title="Python" - validation_definition = context.validation_definitions.add(validation_definition) + ```python title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py - create a Validation Definition" ``` - :::tip - - You can add a Validation Definition to your Data Context at the same time as you create it with the following code: +4. Optional. Save the Validation Definition to your Data Context. - ```python title="Python" - definition_name = "My second Validation Definition" - validation_definition = context.validation_definitions.add(ValidationDefinition(data=batch_definition, suite=suite, name=definition_name)) + ```python title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py - save the Validation Definition to the Data Context" ``` - ::: - -```python showLineNumbers title="Python" -from great_expectations.core import ValidationDefinition -import great_expectations as gx - -context = gx.get_context() - -suite_name = "my_expectation_suite" -suite = context.suites.get(name=suite_name) - -data_source_name = "my_datasource" -data_asset_name = "my_data_asset" -batch_definition_name = "my_batch_definition" -batch_definition = context.get_datasource(data_source_name).get_asset(data_asset_name).get_batch_definition(batch_definition_name) - -# highlight-start -definition_name = "My Validation Definition" -validation_definition = ValidationDefinition(data=batch_definition, suite=suite, name=definition_name) -# highlight-end - -# highlight-start -validation_definition = context.validation_definitions.add(validation_definition) -# highlight-end - -# highlight-start -new_definition_name = "My second Validation Definition" -validation_definition = context.validation_definitions.add(ValidationDefinition(data=batch_definition, suite=suite, name=new_definition_name)) -# highlight-end +```python showLineNumbers title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/create_a_validation_definition.py - full code example" ``` diff --git a/docs/docusaurus/docs/core/run_validations/run_a_validation_definition.md b/docs/docusaurus/docs/core/run_validations/run_a_validation_definition.md index 1062e884d4cc..8f68394b5d40 100644 --- a/docs/docusaurus/docs/core/run_validations/run_a_validation_definition.md +++ b/docs/docusaurus/docs/core/run_validations/run_a_validation_definition.md @@ -15,7 +15,7 @@ import PrereqValidationDefinition from '../_core_components/prerequisites/_valid - . - . -- . +- . In this guide the variable `context` is assumed to contain your Data Context. - . @@ -26,18 +26,12 @@ import PrereqValidationDefinition from '../_core_components/prerequisites/_valid If you have created a new Validation Definition you can use the object returned by your Data Context's `.validation_definitions.add(...)` method. Alternatively, you can retrieve a previously configured Validation Definition by updating the variable `validation_definition_name` in the following code and executing it: - ```python title="Python - import great_expectations as gx - context = gx.get_context() - - validation_definition_name = "my_validation_definition" - validation_definition = context.validation_definitions.get(validation_definition_name) + ```python title="Python name="docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py - retrieve a Validation Definition" ``` 2. Execute the Validation Definition's `run()` method: - ```python title="Python" - validation_result = validation_definition.run() + ```python title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py - run a Validation Definition" ``` Validation Results are automatically saved in your Data Context when a Validation Definition's `run()` method is called. For convenience, the `run()` method also returns the Validation Results as an object you can review. @@ -50,8 +44,7 @@ import PrereqValidationDefinition from '../_core_components/prerequisites/_valid 3. Review the Validation Results: - ```python title="Python" - print(validation_result) + ```python title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py - review Validation Results" ``` When you print the returned Validation Result object you will recieve a yaml representation of the results. By default this will include a `"results"` list that includes each Expectation in your Validation Definition's Expectation Suite, whether the Expectation was successfully met or failed to pass, and some sumarized information explaining the why the Expectation succeeded or failed. @@ -61,7 +54,7 @@ import PrereqValidationDefinition from '../_core_components/prerequisites/_valid GX Cloud users can view the Validation Results in the GX Cloud UI by following the url provided with: ```python title="Python" - print(validation_result.result_url) + print(validation_results.results_url) ``` ::: @@ -70,22 +63,7 @@ import PrereqValidationDefinition from '../_core_components/prerequisites/_valid -```python showLineNumbers title="Python" -import great_expectations as gx - -context = gx.get_context() - -validation_definition_name = "my_validation_definition" -validation_definition = context.validation_definitions.get(validation_definition_name) - -# highlight-next-line -validation_result = validation_definition.run() - -# highlight-next-line -print(validation_result) - -# highlight-next-line -print(validation_result.results_url) +```python showLineNumbers title="Python" name="docs/docusaurus/docs/core/run_validations/_examples/run_a_validation_definition.py - full code example" ``` diff --git a/docs/docusaurus/docusaurus.config.js b/docs/docusaurus/docusaurus.config.js index 4c63adcbb0ca..3c3e2839eb3e 100644 --- a/docs/docusaurus/docusaurus.config.js +++ b/docs/docusaurus/docusaurus.config.js @@ -76,6 +76,18 @@ module.exports = { prism: { additionalLanguages: ['bash'], theme: require('./src/theme/CodeBlock/theme'), + magicComments: [ + // Remember to extend the default highlight class name as well! + { + className: 'theme-code-block-highlighted-line', + line: 'highlight-next-line', + block: {start: 'highlight-start', end: 'highlight-end'}, + }, + { + className: 'code-block-hide-line', + line: 'Hide this', + }, + ] }, colorMode: { disableSwitch: true, diff --git a/docs/docusaurus/src/css/custom.scss b/docs/docusaurus/src/css/custom.scss index 6360dda5c26c..f2f3d1988530 100755 --- a/docs/docusaurus/src/css/custom.scss +++ b/docs/docusaurus/src/css/custom.scss @@ -292,3 +292,7 @@ li p + ol { } @import "api_docs/api_docs.scss"; + +.code-block-hide-line { + display:none; +}