From bd357d0fe97beabf6ba14d0917b6146e4f584930 Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Tue, 10 Dec 2024 10:26:02 -0500 Subject: [PATCH 1/4] update docs --- .../public_api_excludes.py | 5 ----- .../public_api_includes.py | 5 ----- great_expectations/checkpoint/actions.py | 22 +++++++++++++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/sphinx_api_docs_source/public_api_excludes.py b/docs/sphinx_api_docs_source/public_api_excludes.py index a3d58d3c497d..a09d55db4fa1 100644 --- a/docs/sphinx_api_docs_source/public_api_excludes.py +++ b/docs/sphinx_api_docs_source/public_api_excludes.py @@ -106,11 +106,6 @@ "great_expectations/datasource/batch_kwargs_generator/table_batch_kwargs_generator.py" ), ), - IncludeExcludeDefinition( - reason="ValidationActions are now run from Checkpoints: https://docs.greatexpectations.io/docs/guides/miscellaneous/migration_guide#manually-migrate-v2-checkpoints-to-v3-checkpoints", - name="run", - filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), - ), IncludeExcludeDefinition( reason="CLI internal methods should not be part of the public API", filepath=pathlib.Path("great_expectations/cli/datasource.py"), diff --git a/docs/sphinx_api_docs_source/public_api_includes.py b/docs/sphinx_api_docs_source/public_api_includes.py index dee89e1c6e9f..d5768de4eed8 100644 --- a/docs/sphinx_api_docs_source/public_api_includes.py +++ b/docs/sphinx_api_docs_source/public_api_includes.py @@ -24,11 +24,6 @@ name="run", filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), ), - IncludeExcludeDefinition( - reason="Validation Actions are used within Checkpoints but are part of our Public API and can be overridden via plugins.", - name="_run", - filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), - ), IncludeExcludeDefinition( reason="Validation Actions are used within Checkpoints but are part of our Public API and can be overridden via plugins.", name="SlackNotificationAction", diff --git a/great_expectations/checkpoint/actions.py b/great_expectations/checkpoint/actions.py index 0ff1321a29a8..17cc44dc27df 100644 --- a/great_expectations/checkpoint/actions.py +++ b/great_expectations/checkpoint/actions.py @@ -83,6 +83,7 @@ def _build_renderer(config: dict) -> Renderer: return renderer +@public_api class ActionContext: """ Shared context for all actions in a checkpoint run. @@ -99,7 +100,17 @@ def data(self) -> list[tuple[ValidationAction, dict]]: def update(self, action: ValidationAction, action_result: dict) -> None: self._data.append((action, action_result)) + @public_api def filter_results(self, class_: Type[ValidationAction]) -> list[dict]: + """ + Filter the results of the actions in the context by class. + + Args: + class_: The class to filter by. + + Returns: + A list of action results. + """ return [action_result for action, action_result in self._data if isinstance(action, class_)] @@ -125,9 +136,20 @@ class Config: def _using_cloud_context(self) -> bool: return project_manager.is_using_cloud() + @public_api def run( self, checkpoint_result: CheckpointResult, action_context: ActionContext | None = None ) -> dict: + """ + Run the action. + + Args: + checkpoint_result: The result of the checkpoint run. + action_context: The context in which the action is run. + + Returns: + A dictionary containing the result of the action. + """ raise NotImplementedError def _get_data_docs_pages_from_prior_action( From e0893c214c2ff968641de9d71776600e6de8f617 Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Tue, 10 Dec 2024 10:37:40 -0500 Subject: [PATCH 2/4] update test --- docs/sphinx_api_docs_source/public_api_missing_threshold.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sphinx_api_docs_source/public_api_missing_threshold.py b/docs/sphinx_api_docs_source/public_api_missing_threshold.py index faaa1a76017e..61020f5d4753 100644 --- a/docs/sphinx_api_docs_source/public_api_missing_threshold.py +++ b/docs/sphinx_api_docs_source/public_api_missing_threshold.py @@ -6,7 +6,6 @@ ITEMS_IGNORED_FROM_PUBLIC_API = [ "File: great_expectations/_docs_decorators.py Name: add", - "File: great_expectations/checkpoint/actions.py Name: _run", "File: great_expectations/checkpoint/actions.py Name: run", "File: great_expectations/checkpoint/actions.py Name: update", "File: great_expectations/checkpoint/checkpoint.py Name: describe_dict", From bd1fd1f05150c4b245b9623ccafb3602b5b9004e Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Wed, 11 Dec 2024 15:23:37 -0500 Subject: [PATCH 3/4] update public api --- .../public_api_missing_threshold.py | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/docs/sphinx_api_docs_source/public_api_missing_threshold.py b/docs/sphinx_api_docs_source/public_api_missing_threshold.py index e69de29bb2d1..cbdc526b5e69 100644 --- a/docs/sphinx_api_docs_source/public_api_missing_threshold.py +++ b/docs/sphinx_api_docs_source/public_api_missing_threshold.py @@ -0,0 +1,168 @@ +"""Methods and classes that are not marked with the @public_api decorator but may appear in our public docs are listed here. + +Over time this list should be driven to 0 by either adding the @public_api decorator and an appropriate docstring or +adding an exclude directive to docs/sphinx_api_docs_source/public_api_excludes.py +""" + +ITEMS_IGNORED_FROM_PUBLIC_API = [ + "File: great_expectations/_docs_decorators.py Name: add", + "File: great_expectations/checkpoint/actions.py Name: get", + "File: great_expectations/checkpoint/actions.py Name: run", + "File: great_expectations/checkpoint/actions.py Name: update", + "File: great_expectations/checkpoint/checkpoint.py Name: describe_dict", + "File: great_expectations/compatibility/not_imported.py Name: is_version_greater_or_equal", + "File: great_expectations/compatibility/typing_extensions.py Name: override", + "File: great_expectations/core/batch.py Name: head", + "File: great_expectations/core/batch_definition.py Name: build_batch_request", + "File: great_expectations/core/expectation_diagnostics/expectation_doctor.py Name: print_diagnostic_checklist", + "File: great_expectations/core/expectation_diagnostics/expectation_doctor.py Name: run_diagnostics", + "File: great_expectations/core/expectation_suite.py Name: add_expectation_configuration", + "File: great_expectations/core/expectation_suite.py Name: remove_expectation", + "File: great_expectations/core/expectation_validation_result.py Name: describe_dict", + "File: great_expectations/core/factory/factory.py Name: add", + "File: great_expectations/core/factory/factory.py Name: all", + "File: great_expectations/core/factory/factory.py Name: get", + "File: great_expectations/core/metric_domain_types.py Name: MetricDomainTypes", + "File: great_expectations/core/metric_function_types.py Name: MetricPartialFunctionTypes", + "File: great_expectations/core/partitioners.py Name: ColumnPartitionerMonthly", + "File: great_expectations/core/partitioners.py Name: PartitionerColumnValue", + "File: great_expectations/core/yaml_handler.py Name: YAMLHandler", + "File: great_expectations/core/yaml_handler.py Name: dump", + "File: great_expectations/core/yaml_handler.py Name: load", + "File: great_expectations/data_context/data_context/abstract_data_context.py Name: add_store", + "File: great_expectations/data_context/data_context/abstract_data_context.py Name: delete_datasource", + "File: great_expectations/data_context/data_context/abstract_data_context.py Name: get_docs_sites_urls", + "File: great_expectations/data_context/data_context/abstract_data_context.py Name: get_validator", + "File: great_expectations/data_context/data_context/abstract_data_context.py Name: list_datasources", + "File: great_expectations/data_context/data_context/abstract_data_context.py Name: open_data_docs", + "File: great_expectations/data_context/data_context/context_factory.py Name: build_data_docs", + "File: great_expectations/data_context/data_context/context_factory.py Name: get_context", + "File: great_expectations/data_context/data_context/context_factory.py Name: get_docs_sites_urls", + "File: great_expectations/data_context/data_context/context_factory.py Name: get_validator", + "File: great_expectations/data_context/data_context_variables.py Name: save", + "File: great_expectations/data_context/store/_store_backend.py Name: add", + "File: great_expectations/data_context/store/_store_backend.py Name: update", + "File: great_expectations/data_context/store/checkpoint_store.py Name: CheckpointStore", + "File: great_expectations/data_context/store/database_store_backend.py Name: DatabaseStoreBackend", + "File: great_expectations/data_context/store/expectations_store.py Name: ExpectationsStore", + "File: great_expectations/data_context/store/metric_store.py Name: MetricStore", + "File: great_expectations/data_context/store/query_store.py Name: SqlAlchemyQueryStore", + "File: great_expectations/data_context/store/store.py Name: add", + "File: great_expectations/data_context/store/store.py Name: update", + "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleAzureBlobStoreBackend", + "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleFilesystemStoreBackend", + "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleGCSStoreBackend", + "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleS3StoreBackend", + "File: great_expectations/data_context/store/validation_definition_store.py Name: ValidationDefinitionStore", + "File: great_expectations/data_context/store/validation_results_store.py Name: ValidationResultsStore", + "File: great_expectations/data_context/types/base.py Name: update", + "File: great_expectations/data_context/types/resource_identifiers.py Name: GXCloudIdentifier", + "File: great_expectations/datasource/datasource_dict.py Name: add_dataframe_asset", + "File: great_expectations/datasource/fluent/config.py Name: yaml", + "File: great_expectations/datasource/fluent/config_str.py Name: ConfigStr", + "File: great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py Name: columns", + "File: great_expectations/datasource/fluent/data_asset/path/directory_asset.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/data_asset/path/directory_asset.py Name: get_batch_parameters_keys", + "File: great_expectations/datasource/fluent/data_asset/path/file_asset.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/data_asset/path/file_asset.py Name: get_batch_parameters_keys", + "File: great_expectations/datasource/fluent/data_asset/path/path_data_asset.py Name: get_batch", + "File: great_expectations/datasource/fluent/data_asset/path/path_data_asset.py Name: get_batch_parameters_keys", + "File: great_expectations/datasource/fluent/data_connector/batch_filter.py Name: validate", + "File: great_expectations/datasource/fluent/fabric.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/fabric.py Name: get_batch", + "File: great_expectations/datasource/fluent/fluent_base_model.py Name: yaml", + "File: great_expectations/datasource/fluent/invalid_datasource.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/invalid_datasource.py Name: get_asset", + "File: great_expectations/datasource/fluent/invalid_datasource.py Name: get_batch", + "File: great_expectations/datasource/fluent/invalid_datasource.py Name: get_batch_parameters_keys", + "File: great_expectations/datasource/fluent/pandas_datasource.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/pandas_datasource.py Name: get_batch", + "File: great_expectations/datasource/fluent/sources.py Name: delete_datasource", + "File: great_expectations/datasource/fluent/spark_datasource.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/spark_datasource.py Name: get_batch", + "File: great_expectations/datasource/fluent/sql_datasource.py Name: build_batch_request", + "File: great_expectations/datasource/fluent/sql_datasource.py Name: get_batch", + "File: great_expectations/exceptions/exceptions.py Name: DataContextError", + "File: great_expectations/exceptions/exceptions.py Name: InvalidExpectationConfigurationError", + "File: great_expectations/execution_engine/execution_engine.py Name: ExecutionEngine", + "File: great_expectations/execution_engine/execution_engine.py Name: get_compute_domain", + "File: great_expectations/execution_engine/pandas_execution_engine.py Name: PandasExecutionEngine", + "File: great_expectations/execution_engine/pandas_execution_engine.py Name: get_compute_domain", + "File: great_expectations/execution_engine/sparkdf_execution_engine.py Name: SparkDFExecutionEngine", + "File: great_expectations/execution_engine/sparkdf_execution_engine.py Name: get_compute_domain", + "File: great_expectations/execution_engine/sqlalchemy_execution_engine.py Name: SqlAlchemyExecutionEngine", + "File: great_expectations/execution_engine/sqlalchemy_execution_engine.py Name: execute_query", + "File: great_expectations/execution_engine/sqlalchemy_execution_engine.py Name: get_compute_domain", + "File: great_expectations/expectations/core/expect_column_max_to_be_between.py Name: ExpectColumnMaxToBeBetween", + "File: great_expectations/expectations/core/expect_column_to_exist.py Name: ExpectColumnToExist", + "File: great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py Name: ExpectColumnValuesToBeInTypeList", + "File: great_expectations/expectations/core/expect_column_values_to_be_null.py Name: ExpectColumnValuesToBeNull", + "File: great_expectations/expectations/core/expect_column_values_to_be_of_type.py Name: ExpectColumnValuesToBeOfType", + "File: great_expectations/expectations/core/expect_table_column_count_to_be_between.py Name: ExpectTableColumnCountToBeBetween", + "File: great_expectations/expectations/core/expect_table_column_count_to_equal.py Name: ExpectTableColumnCountToEqual", + "File: great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py Name: ExpectTableColumnsToMatchOrderedList", + "File: great_expectations/expectations/core/expect_table_columns_to_match_set.py Name: ExpectTableColumnsToMatchSet", + "File: great_expectations/expectations/core/expect_table_row_count_to_be_between.py Name: ExpectTableRowCountToBeBetween", + "File: great_expectations/expectations/core/expect_table_row_count_to_equal.py Name: ExpectTableRowCountToEqual", + "File: great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py Name: ExpectTableRowCountToEqualOtherTable", + "File: great_expectations/expectations/core/unexpected_rows_expectation.py Name: UnexpectedRowsExpectation", + "File: great_expectations/expectations/expectation.py Name: ColumnAggregateExpectation", + "File: great_expectations/expectations/expectation.py Name: ColumnMapExpectation", + "File: great_expectations/expectations/expectation.py Name: UnexpectedRowsExpectation", + "File: great_expectations/expectations/expectation.py Name: render_suite_parameter_string", + "File: great_expectations/expectations/expectation.py Name: validate_configuration", + "File: great_expectations/expectations/expectation_configuration.py Name: ExpectationConfiguration", + "File: great_expectations/expectations/expectation_configuration.py Name: to_domain_obj", + "File: great_expectations/expectations/expectation_configuration.py Name: type", + "File: great_expectations/expectations/metrics/column_aggregate_metric_provider.py Name: ColumnAggregateMetricProvider", + "File: great_expectations/expectations/metrics/column_aggregate_metric_provider.py Name: column_aggregate_partial", + "File: great_expectations/expectations/metrics/column_aggregate_metric_provider.py Name: column_aggregate_value", + "File: great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py Name: column_condition_partial", + "File: great_expectations/expectations/metrics/map_metric_provider/column_map_metric_provider.py Name: ColumnMapMetricProvider", + "File: great_expectations/expectations/metrics/metric_provider.py Name: MetricProvider", + "File: great_expectations/expectations/metrics/metric_provider.py Name: metric_partial", + "File: great_expectations/expectations/metrics/metric_provider.py Name: metric_value", + "File: great_expectations/expectations/regex_based_column_map_expectation.py Name: validate_configuration", + "File: great_expectations/expectations/set_based_column_map_expectation.py Name: validate_configuration", + "File: great_expectations/experimental/metric_repository/metric_retriever.py Name: get_validator", + "File: great_expectations/experimental/rule_based_profiler/helpers/util.py Name: build_batch_request", + "File: great_expectations/experimental/rule_based_profiler/rule_based_profiler.py Name: run", + "File: great_expectations/render/components.py Name: CollapseContent", + "File: great_expectations/render/components.py Name: RenderedStringTemplateContent", + "File: great_expectations/render/components.py Name: RenderedTableContent", + "File: great_expectations/render/components.py Name: validate", + "File: great_expectations/render/renderer/email_renderer.py Name: EmailRenderer", + "File: great_expectations/render/renderer/microsoft_teams_renderer.py Name: MicrosoftTeamsRenderer", + "File: great_expectations/render/renderer/opsgenie_renderer.py Name: OpsgenieRenderer", + "File: great_expectations/render/renderer/renderer.py Name: renderer", + "File: great_expectations/render/renderer/site_builder.py Name: DefaultSiteIndexBuilder", + "File: great_expectations/render/renderer/site_builder.py Name: SiteBuilder", + "File: great_expectations/render/renderer/slack_renderer.py Name: SlackRenderer", + "File: great_expectations/render/util.py Name: handle_strict_min_max", + "File: great_expectations/render/util.py Name: num_to_str", + "File: great_expectations/render/util.py Name: parse_row_condition_string_pandas_engine", + "File: great_expectations/render/util.py Name: substitute_none_for_missing", + "File: great_expectations/validator/metric_configuration.py Name: MetricConfiguration", + "File: great_expectations/validator/metrics_calculator.py Name: columns", + "File: great_expectations/validator/validation_graph.py Name: resolve", + "File: great_expectations/validator/validator.py Name: columns", + "File: great_expectations/validator/validator.py Name: head", + "File: great_expectations/validator/validator.py Name: remove_expectation", + "File: great_expectations/validator/validator.py Name: save_expectation_suite", + "File: great_expectations/validator/validator.py Name: validate", + # Expectations referenced in the Learn data quality use cases: + "File: great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py Name: ExpectColumnKLDivergenceToBeLessThan", + "File: great_expectations/expectations/core/expect_column_mean_to_be_between.py Name: ExpectColumnMeanToBeBetween", + "File: great_expectations/expectations/core/expect_column_median_to_be_between.py Name: ExpectColumnMedianToBeBetween", + "File: great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py Name: ExpectColumnQuantileValuesToBeBetween", + "File: great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py Name: ExpectColumnValueZScoresToBeLessThan", + "File: great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py Name: ExpectColumnPairValuesAToBeGreaterThanB", + "File: great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py Name: ExpectColumnPairValuesToBeEqual", + "File: great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py Name: ExpectMulticolumnSumToEqual", + "File: great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py Name: ExpectColumnProportionOfUniqueValuesToBeBetween", + "File: great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py Name: ExpectColumnUniqueValueCountToBeBetween", + "File: great_expectations/expectations/core/expect_column_values_to_be_unique.py Name: ExpectColumnValuesToBeUnique", + "File: great_expectations/expectations/core/expect_compound_columns_to_be_unique.py Name: ExpectCompoundColumnsToBeUnique", + "File: great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py Name: ExpectSelectColumnValuesToBeUniqueWithinRecord", + "File: great_expectations/expectations/core/expect_column_min_to_be_between.py Name: ExpectColumnMinToBeBetween", +] From 9ba15a69fcaaaa10aa35918d918fbdbbc8af3416 Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Wed, 11 Dec 2024 16:16:19 -0500 Subject: [PATCH 4/4] more public api --- docs/sphinx_api_docs_source/public_api_includes.py | 5 +++++ docs/sphinx_api_docs_source/public_api_missing_threshold.py | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/sphinx_api_docs_source/public_api_includes.py b/docs/sphinx_api_docs_source/public_api_includes.py index d5768de4eed8..dee89e1c6e9f 100644 --- a/docs/sphinx_api_docs_source/public_api_includes.py +++ b/docs/sphinx_api_docs_source/public_api_includes.py @@ -24,6 +24,11 @@ name="run", filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), ), + IncludeExcludeDefinition( + reason="Validation Actions are used within Checkpoints but are part of our Public API and can be overridden via plugins.", + name="_run", + filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), + ), IncludeExcludeDefinition( reason="Validation Actions are used within Checkpoints but are part of our Public API and can be overridden via plugins.", name="SlackNotificationAction", diff --git a/docs/sphinx_api_docs_source/public_api_missing_threshold.py b/docs/sphinx_api_docs_source/public_api_missing_threshold.py index cbdc526b5e69..7004b05f676e 100644 --- a/docs/sphinx_api_docs_source/public_api_missing_threshold.py +++ b/docs/sphinx_api_docs_source/public_api_missing_threshold.py @@ -6,6 +6,7 @@ ITEMS_IGNORED_FROM_PUBLIC_API = [ "File: great_expectations/_docs_decorators.py Name: add", + "File: great_expectations/checkpoint/actions.py Name: _run", "File: great_expectations/checkpoint/actions.py Name: get", "File: great_expectations/checkpoint/actions.py Name: run", "File: great_expectations/checkpoint/actions.py Name: update",