From 2fda29d1a38c82989db6e126461422915759fb1e Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Tue, 17 Dec 2024 14:06:46 -0500 Subject: [PATCH 1/4] [MAINTENANCE] Remove GX Cloud onboarding script (#10785) --- .../experimental/onboarding_script.py | 115 ------------------ 1 file changed, 115 deletions(-) delete mode 100644 assets/scripts/gx_cloud/experimental/onboarding_script.py diff --git a/assets/scripts/gx_cloud/experimental/onboarding_script.py b/assets/scripts/gx_cloud/experimental/onboarding_script.py deleted file mode 100644 index 508b768cb718..000000000000 --- a/assets/scripts/gx_cloud/experimental/onboarding_script.py +++ /dev/null @@ -1,115 +0,0 @@ -import pprint - -import great_expectations as gx -from great_expectations.checkpoint import Checkpoint -from great_expectations.core.expectation_suite import ExpectationSuite -from great_expectations.data_context import CloudDataContext -from great_expectations.datasource.fluent import BatchRequest, Datasource -from great_expectations.datasource.fluent.pandas_datasource import CSVAsset - -# Make sure GX_CLOUD_ACCESS_TOKEN and GX_CLOUD_ORGANIZATION_ID -# are set in your environment or config_variables.yml -# your organization_id is indicated on https://app.greatexpectations.io/tokens page - -# uncomment the next three lines to set them explicitly in this script -# import os -# os.environ["GX_CLOUD_ACCESS_TOKEN"] = "" -# os.environ["GX_CLOUD_ORGANIZATION_ID"] = "" - -# Create a GX Data Context -context: CloudDataContext = gx.get_context( - cloud_mode=True, -) - -# Provide Datasource name -datasource_name = None -assert datasource_name, "Please set datasource_name." - -# Get or add Datasource -datasource: Datasource = context.data_sources.add_or_update_pandas(datasource_name) - -# Provide an Asset name -asset_name = None -assert asset_name, "Please set asset_name." - -# Provide a path to data -path_to_data = None -# to use sample data uncomment next line -# path_to_data = "https://raw.githubusercontent.com/great-expectations/gx_tutorials/main/data/yellow_tripdata_sample_2019-01.csv" -assert path_to_data, "Please set path_to_data. This can be a local filepath or a remote URL." - -# Get or add Asset -try: - asset: CSVAsset = datasource.get_asset(asset_name=asset_name) -except LookupError: - asset: CSVAsset = datasource.add_csv_asset(asset_name, filepath_or_buffer=path_to_data) - -# Build BatchRequest -batch_request: BatchRequest = asset.build_batch_request() - -print(f"\n{20*'='}\nDatasource Config\n{20*'='}\n") -pprint.pprint(datasource.dict()) - -# Provide an Expectation Suite name -expectation_suite_name = None -assert expectation_suite_name, "Please set expectation_suite_name." - -# Get or add Expectation Suite -expectation_suite: ExpectationSuite = context.suites.add( - ExpectationSuite(name=expectation_suite_name) -) -expectation_suite_ge_cloud_id: str = expectation_suite.ge_cloud_id - -# Add Expectations - -# Set a column name you want to test here -column_name = None -# Uncomment the next line for a column name from sample data -# column_name = "passenger_count" -assert column_name is not None, "Please set column_name." - -# Look up all expectations types here - https://greatexpectations.io/expectations/ -expectation_configuration = gx.core.ExpectationConfiguration( - **{ - "type": "expect_column_min_to_be_between", - "kwargs": {"column": column_name, "min_value": 0.1}, - "meta": {}, - } -) - -expectation_suite.add_expectation_configuration(expectation_configuration=expectation_configuration) - -# Save the Expectation Suite -expectation_suite.save() - -print(f"\n{20*'='}\nExpectation Suite\n{20*'='}\n") -pprint.pprint(expectation_suite) - -# Provide a Checkpoint name -checkpoint_name = None -assert checkpoint_name, "Please set checkpoint_name." - -checkpoint_config = { - "name": checkpoint_name, - "validations": [ - { - "expectation_suite_name": expectation_suite_name, - "expectation_suite_ge_cloud_id": expectation_suite.ge_cloud_id, - "batch_request": { - "datasource_name": datasource.name, - "data_asset_name": asset.name, - }, - } - ], -} - -checkpoint: Checkpoint = context.add_or_update_checkpoint(**checkpoint_config) - -print(f"\n{20*'='}\nCheckpoint Config\n{20*'='}\n") -pprint.pprint(checkpoint) - -# Run the Checkpoint: -result = checkpoint.run() - -print(f"\n{20*'='}\nValidation Result\n{20*'='}\n") -pprint.pprint(result) From 504bf4d23733b31a3f0ac878fb68d4ba681c4427 Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Tue, 17 Dec 2024 14:09:12 -0500 Subject: [PATCH 2/4] [MAINTENANCE] Update `tasks.py` to remove reference to `isort` (#10782) --- tasks.py | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/tasks.py b/tasks.py index f76b3064c15e..bf32692faafd 100644 --- a/tasks.py +++ b/tasks.py @@ -52,11 +52,6 @@ "check": _CHECK_HELP_DESC, "exclude": _EXCLUDE_HELP_DESC, "path": _PATH_HELP_DESC, - "isort": "Use `isort` to sort packages. Default behavior.", - "ruff": ( - "Use `ruff` instead of `isort` to sort imports." - " This will eventually become the default." - ), "pty": _PTY_HELP_DESC, } ) @@ -65,29 +60,18 @@ def sort( path: str = ".", check: bool = False, exclude: str | None = None, - ruff: bool = False, # isort is the current default - isort: bool = False, pty: bool = True, ): """Sort module imports.""" - if ruff and isort: - raise invoke.Exit("cannot use both `--ruff` and `--isort`", code=1) # noqa: TRY003 - if not isort: - cmds = [ - "ruff", - "check", - path, - "--select I", - "--diff" if check else "--fix", - ] - if exclude: - cmds.extend(["--extend-exclude", exclude]) - else: - cmds = ["isort", path] - if check: - cmds.append("--check-only") - if exclude: - cmds.extend(["--skip", exclude]) + cmds = [ + "ruff", + "check", + path, + "--select I", + "--diff" if check else "--fix", + ] + if exclude: + cmds.extend(["--extend-exclude", exclude]) ctx.run(" ".join(cmds), echo=True, pty=pty) From 7308db306db8470393f20bf5458823ab21f87f41 Mon Sep 17 00:00:00 2001 From: Tyler Hoffman Date: Tue, 17 Dec 2024 14:22:52 -0500 Subject: [PATCH 3/4] [BUGFIX] Fix expectation description rendering in DataDocs (#10789) --- .../render/renderer/content_block/content_block.py | 6 +++--- .../validation_results_table_content_block.py | 4 +--- tests/render/test_column_section_renderer.py | 14 +++++++------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/great_expectations/render/renderer/content_block/content_block.py b/great_expectations/render/renderer/content_block/content_block.py index 023efb8cb826..b21ec93e83a3 100644 --- a/great_expectations/render/renderer/content_block/content_block.py +++ b/great_expectations/render/renderer/content_block/content_block.py @@ -297,10 +297,10 @@ def _render_expectation_description( description = expectation.description if not description: raise ValueError("Cannot render an expectation with no description.") # noqa: TRY003 - # If we wish to support $VAR substitution, we should use RenderedStringTemplateContent with params # noqa: E501 return [ - RenderedMarkdownContent( - markdown=description, styling=runtime_configuration.get("styling", {}) + RenderedStringTemplateContent( + string_template={"template": description}, + styling=runtime_configuration.get("styling", {}), ) ] diff --git a/great_expectations/render/renderer/content_block/validation_results_table_content_block.py b/great_expectations/render/renderer/content_block/validation_results_table_content_block.py index 7556ecd586c6..6fe0f66f7d4d 100644 --- a/great_expectations/render/renderer/content_block/validation_results_table_content_block.py +++ b/great_expectations/render/renderer/content_block/validation_results_table_content_block.py @@ -80,7 +80,7 @@ def _process_content_block(cls, content_block, has_failed_evr, render_object=Non @override @classmethod - def _get_content_block_fn( # noqa: C901, PLR0915 + def _get_content_block_fn( # noqa: C901 cls, expectation_type: str, expectation_config: ExpectationConfiguration | None = None, @@ -88,8 +88,6 @@ def _get_content_block_fn( # noqa: C901, PLR0915 content_block_fn = super()._get_content_block_fn( expectation_type=expectation_type, expectation_config=expectation_config ) - if content_block_fn == cls._render_expectation_description: - return content_block_fn expectation_string_fn = content_block_fn if expectation_string_fn is None: diff --git a/tests/render/test_column_section_renderer.py b/tests/render/test_column_section_renderer.py index f99f703d3a64..94a6667e034d 100644 --- a/tests/render/test_column_section_renderer.py +++ b/tests/render/test_column_section_renderer.py @@ -1130,9 +1130,9 @@ def test_ExpectationSuiteColumnSectionRenderer_render_expectation_with_descripti content_block = result.content_blocks[1] content = content_block.bullet_list[0] - markdown = content.markdown - - assert markdown == expectation.description + assert content.string_template == { + "template": "column values must be a legal adult age (**18** or older)" + } @pytest.mark.unit @@ -1588,10 +1588,10 @@ def test_ValidationResultsTableContentBlockRenderer_render_evr_with_description( result = ValidationResultsColumnSectionRenderer().render([evr]) content_block = result.content_blocks[1] - content = content_block.table[0] - markdown = content.markdown - - assert markdown == expectation.description + _, description_cell, _ = content_block.table[0] + assert description_cell.string_template == { + "template": "column values must be a legal adult age (**18** or older)" + } # noinspection PyPep8Naming From 46c856974bb67caa124e5d5f64bbe2af8759d549 Mon Sep 17 00:00:00 2001 From: Chetan Kini Date: Tue, 17 Dec 2024 14:29:23 -0500 Subject: [PATCH 4/4] [MAINTENANCE] Check filepath existence when evaluating public API report (#10754) --- .../include_exclude_definition.py | 2 + .../printable_definition.py | 20 + .../public_api_excludes.py | 258 +---- .../public_api_includes.py | 152 --- .../public_api_missing_threshold.py | 936 +++++++++++++++--- .../public_api_report.py | 39 +- tests/scripts/test_public_api_report.py | 159 ++- 7 files changed, 887 insertions(+), 679 deletions(-) create mode 100644 docs/sphinx_api_docs_source/printable_definition.py diff --git a/docs/sphinx_api_docs_source/include_exclude_definition.py b/docs/sphinx_api_docs_source/include_exclude_definition.py index 5985f6bed08f..77ebaae92600 100644 --- a/docs/sphinx_api_docs_source/include_exclude_definition.py +++ b/docs/sphinx_api_docs_source/include_exclude_definition.py @@ -31,3 +31,5 @@ def __post_init__(self): raise ValueError( # noqa: TRY003 "You must provide at least a filepath or filepath and name." ) + if self.filepath and not self.filepath.exists(): + raise FileNotFoundError(f"File {self.filepath} does not exist.") # noqa: TRY003 diff --git a/docs/sphinx_api_docs_source/printable_definition.py b/docs/sphinx_api_docs_source/printable_definition.py new file mode 100644 index 000000000000..866fdee071f3 --- /dev/null +++ b/docs/sphinx_api_docs_source/printable_definition.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import pathlib +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PrintableDefinition: + file: pathlib.Path + name: str + + def __post_init__(self): + if not self.file.exists(): + raise FileNotFoundError(f"File {self.file} does not exist.") # noqa: TRY003 + + def __str__(self) -> str: # type: ignore[explicit-override] + return f"File: {self.file} Name: {self.name}" + + def __lt__(self, other) -> bool: + return str(self) < str(other) diff --git a/docs/sphinx_api_docs_source/public_api_excludes.py b/docs/sphinx_api_docs_source/public_api_excludes.py index a3d58d3c497d..13648c5ae9f0 100644 --- a/docs/sphinx_api_docs_source/public_api_excludes.py +++ b/docs/sphinx_api_docs_source/public_api_excludes.py @@ -13,20 +13,6 @@ ) DEFAULT_EXCLUDES: list[IncludeExcludeDefinition] = [ - IncludeExcludeDefinition( - reason="We now use get_context(), this method only exists for backward compatibility.", - name="DataContext", - filepath=pathlib.Path( - "great_expectations/data_context/data_context/data_context.py" - ), - ), - IncludeExcludeDefinition( - reason="We now use get_context(), this method only exists for backward compatibility.", - name="BaseDataContext", - filepath=pathlib.Path( - "great_expectations/data_context/data_context/base_data_context.py" - ), - ), IncludeExcludeDefinition( reason="Fluent is not part of the public API", filepath=pathlib.Path("great_expectations/datasource/fluent/interfaces.py"), @@ -52,85 +38,16 @@ reason="Exclude code from __init__.py", filepath=pathlib.Path("great_expectations/types/__init__.py"), ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/databricks_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/glob_reader_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/manual_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/query_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/s3_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/s3_subdir_reader_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/subdir_reader_batch_kwargs_generator.py" - ), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - filepath=pathlib.Path( - "great_expectations/datasource/batch_kwargs_generator/table_batch_kwargs_generator.py" - ), - ), IncludeExcludeDefinition( reason="ValidationActions are now run from Checkpoints: https://docs.greatexpectations.io/docs/guides/miscellaneous/migration_guide#manually-migrate-v2-checkpoints-to-v3-checkpoints", name="run", filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), ), - IncludeExcludeDefinition( - reason="CLI internal methods should not be part of the public API", - filepath=pathlib.Path("great_expectations/cli/datasource.py"), - ), - IncludeExcludeDefinition( - reason="CLI internal methods should not be part of the public API", - filepath=pathlib.Path("great_expectations/cli/toolkit.py"), - ), IncludeExcludeDefinition( reason="False match for from datasource_configuration_test_utilities import is_subset", name="is_subset", filepath=pathlib.Path("great_expectations/core/domain.py"), ), - IncludeExcludeDefinition( - reason="Already captured in the Data Context", - name="test_yaml_config", - filepath=pathlib.Path( - "great_expectations/data_context/config_validator/yaml_config_validator.py" - ), - ), IncludeExcludeDefinition( reason="False match for validator.get_metric()", name="get_metric", @@ -138,28 +55,6 @@ "great_expectations/core/expectation_validation_result.py" ), ), - IncludeExcludeDefinition( - reason="False match for context.suites.get()", - name="get_expectation_suite", - filepath=pathlib.Path("great_expectations/data_asset/data_asset.py"), - ), - IncludeExcludeDefinition( - reason="False match for context.save_expectation_suite() and validator.save_expectation_suite()", - name="save_expectation_suite", - filepath=pathlib.Path("great_expectations/data_asset/data_asset.py"), - ), - IncludeExcludeDefinition( - reason="False match for validator.validate()", - name="validate", - filepath=pathlib.Path("great_expectations/data_asset/data_asset.py"), - ), - IncludeExcludeDefinition( - reason="False match for validator.validate()", - name="validate", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/batch_filter.py" - ), - ), IncludeExcludeDefinition( reason="Captured in AbstractDataContext", name="add_checkpoint", @@ -335,85 +230,6 @@ name="file_relative_path", filepath=pathlib.Path("great_expectations/data_context/util.py"), ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_be_between", - filepath=pathlib.Path("great_expectations/dataset/dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_not_be_null", - filepath=pathlib.Path("great_expectations/dataset/dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_table_row_count_to_be_between", - filepath=pathlib.Path("great_expectations/dataset/dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_be_between", - filepath=pathlib.Path("great_expectations/dataset/pandas_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_not_be_null", - filepath=pathlib.Path("great_expectations/dataset/pandas_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_be_between", - filepath=pathlib.Path("great_expectations/dataset/sparkdf_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_not_be_null", - filepath=pathlib.Path("great_expectations/dataset/sparkdf_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="head", - filepath=pathlib.Path("great_expectations/dataset/sparkdf_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_be_between", - filepath=pathlib.Path("great_expectations/dataset/sqlalchemy_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="expect_column_values_to_not_be_null", - filepath=pathlib.Path("great_expectations/dataset/sqlalchemy_dataset.py"), - ), - IncludeExcludeDefinition( - reason="Exclude code from v2 API", - name="head", - filepath=pathlib.Path("great_expectations/dataset/sqlalchemy_dataset.py"), - ), - IncludeExcludeDefinition( - reason="self_check is mentioned but in the docs we currently recommend using test_yaml_config which uses self_check under the hood. E.g. https://docs.greatexpectations.io/docs/guides/setup/configuring_data_contexts/how_to_configure_datacontext_components_using_test_yaml_config/#steps", - name="self_check", - filepath=pathlib.Path("great_expectations/checkpoint/checkpoint.py"), - ), - IncludeExcludeDefinition( - reason="self_check is mentioned but in the docs we currently recommend using test_yaml_config which uses self_check under the hood. E.g. https://docs.greatexpectations.io/docs/guides/setup/configuring_data_contexts/how_to_configure_datacontext_components_using_test_yaml_config/#steps", - name="self_check", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="self_check is mentioned but in the docs we currently recommend using test_yaml_config which uses self_check under the hood. E.g. https://docs.greatexpectations.io/docs/guides/setup/configuring_data_contexts/how_to_configure_datacontext_components_using_test_yaml_config/#steps", - name="self_check", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/runtime_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="self_check is mentioned but in the docs we currently recommend using test_yaml_config which uses self_check under the hood. E.g. https://docs.greatexpectations.io/docs/guides/setup/configuring_data_contexts/how_to_configure_datacontext_components_using_test_yaml_config/#steps", - name="self_check", - filepath=pathlib.Path("great_expectations/datasource/new_datasource.py"), - ), IncludeExcludeDefinition( reason="False match for dict `.update()` method.", name="update", @@ -522,31 +338,6 @@ name="IDDict", filepath=pathlib.Path("great_expectations/core/id_dict.py"), ), - IncludeExcludeDefinition( - reason="v2 API", - name="expect_column_mean_to_be_between", - filepath=pathlib.Path("great_expectations/dataset/dataset.py"), - ), - IncludeExcludeDefinition( - reason="v2 API", - name="expect_column_values_to_be_in_set", - filepath=pathlib.Path("great_expectations/dataset/dataset.py"), - ), - IncludeExcludeDefinition( - reason="v2 API", - name="expect_column_values_to_be_in_set", - filepath=pathlib.Path("great_expectations/dataset/pandas_dataset.py"), - ), - IncludeExcludeDefinition( - reason="v2 API", - name="expect_column_values_to_be_in_set", - filepath=pathlib.Path("great_expectations/dataset/sparkdf_dataset.py"), - ), - IncludeExcludeDefinition( - reason="v2 API", - name="expect_column_values_to_be_in_set", - filepath=pathlib.Path("great_expectations/dataset/sqlalchemy_dataset.py"), - ), IncludeExcludeDefinition( reason="to_json_dict is an internal helper method", name="to_json_dict", @@ -642,17 +433,6 @@ name="to_json_dict", filepath=pathlib.Path("great_expectations/validator/exception_info.py"), ), - IncludeExcludeDefinition( - reason="False match for DataAssistant.run()", - name="run", - filepath=pathlib.Path( - "great_expectations/experimental/rule_based_profiler/data_assistant/data_assistant_runner.py" - ), - ), - IncludeExcludeDefinition( - reason="Deprecated v2 api Dataset is not included in the public API", - filepath=pathlib.Path("great_expectations/dataset/dataset.py"), - ), IncludeExcludeDefinition( reason="Validate method on custom type not included in the public API", name="validate", @@ -665,13 +445,6 @@ name="columns", filepath=pathlib.Path("great_expectations/datasource/fluent/sql_datasource.py"), ), - IncludeExcludeDefinition( - reason='The "columns()" property in this module is not included in the public API', - name="columns", - filepath=pathlib.Path( - "great_expectations/datasource/fluent/spark_generic_partitioners.py" - ), - ), IncludeExcludeDefinition( reason="The add method shares a name with a public API method", name="add", @@ -703,14 +476,7 @@ IncludeExcludeDefinition( reason="Internal protocols are not included in the public API.", name="add_dataframe_asset", - filepath=pathlib.Path("great_expectations/core/datasource_dict.py"), - ), - IncludeExcludeDefinition( - reason="This method shares a name with a public API method.", - name="get_validator", - filepath=pathlib.Path( - "great_expectations/experimental/metric_repository/column_descriptive_metrics_metric_retriever.py" - ), + filepath=pathlib.Path("great_expectations/datasource/datasource_dict.py"), ), IncludeExcludeDefinition( reason="Not yet part of the public API", @@ -722,11 +488,6 @@ name="ResultFormat", filepath=pathlib.Path("great_expectations/core/result_format.py"), ), - IncludeExcludeDefinition( - reason="Not yet part of the public API, under active development", - name="BatchDefinition", - filepath=pathlib.Path("great_expectations/core/batch_config.py"), - ), IncludeExcludeDefinition( reason="This method shares a name with a public API method.", name="add_expectation", @@ -741,16 +502,6 @@ "great_expectations/data_context/store/expectations_store.py" ), ), - IncludeExcludeDefinition( - reason="This method shares a name with a public API method.", - name="build_batch_request", - filepath=pathlib.Path("great_expectations/core/batch_config.py"), - ), - IncludeExcludeDefinition( - reason="This method shares a name with a public API method.", - name="save", - filepath=pathlib.Path("great_expectations/core/batch_config.py"), - ), IncludeExcludeDefinition( reason="This method shares a name with a public API method.", name="delete", @@ -801,13 +552,6 @@ name="get_or_create_spark_session", filepath=pathlib.Path("great_expectations/core/util.py"), ), - IncludeExcludeDefinition( - reason="This method does not need to be accessed by users, and will eventually be removed from docs.", - name="get_batch_parameters_keys", - filepath=pathlib.Path( - "great_expectations/datasource/fluent/file_path_data_asset.py" - ), - ), IncludeExcludeDefinition( reason="This method does not need to be accessed by users, and will eventually be removed from docs.", name="get_batch_parameters_keys", diff --git a/docs/sphinx_api_docs_source/public_api_includes.py b/docs/sphinx_api_docs_source/public_api_includes.py index dee89e1c6e9f..209581ebb4c1 100644 --- a/docs/sphinx_api_docs_source/public_api_includes.py +++ b/docs/sphinx_api_docs_source/public_api_includes.py @@ -54,144 +54,6 @@ name="UpdateDataDocsAction", filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), ), - IncludeExcludeDefinition( - reason="Validation Actions are used within Checkpoints but are part of our Public API and can be overridden via plugins.", - name="CloudNotificationAction", - filepath=pathlib.Path("great_expectations/checkpoint/actions.py"), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetFilePathDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_file_path_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetDBFSDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_dbfs_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetGCSDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_gcs_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetDBFSDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_dbfs_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="FilePathDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/file_path_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetAzureDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_azure_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="RuntimeDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/runtime_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetAWSGlueDataCatalogDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_aws_glue_data_catalog_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetS3DataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_s3_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetFilesystemDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_filesystem_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetS3DataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_s3_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetAzureDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_azure_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetSqlDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_sql_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetSqlDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_sql_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="DataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetFilePathDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_file_path_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetAWSGlueDataCatalogDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_aws_glue_data_catalog_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="InferredAssetFilesystemDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/inferred_asset_filesystem_data_connector.py" - ), - ), - IncludeExcludeDefinition( - reason="DataConnectors are part of the public API", - name="ConfiguredAssetGCSDataConnector", - filepath=pathlib.Path( - "great_expectations/datasource/data_connector/configured_asset_gcs_data_connector.py" - ), - ), IncludeExcludeDefinition( reason="Data Context types are part of the public API", name="EphemeralDataContext", @@ -213,13 +75,6 @@ "great_expectations/data_context/data_context/cloud_data_context.py" ), ), - IncludeExcludeDefinition( - reason="Map metric providers are part of the public API", - name="MapMetricProvider", - filepath=pathlib.Path( - "great_expectations/expectations/metrics/map_metric_provider.py" - ), - ), IncludeExcludeDefinition( reason="Map metric providers are part of the public API", name="MetricProvider", @@ -227,11 +82,4 @@ "great_expectations/expectations/metrics/metric_provider.py" ), ), - IncludeExcludeDefinition( - reason="Checkpoint CRUD is part of the public API", - name="delete_checkpoint", - filepath=pathlib.Path( - "great_expectations/data_context/data_context/abstract_data_context.py" - ), - ), ] diff --git a/docs/sphinx_api_docs_source/public_api_missing_threshold.py b/docs/sphinx_api_docs_source/public_api_missing_threshold.py index 7004b05f676e..d1c61827131f 100644 --- a/docs/sphinx_api_docs_source/public_api_missing_threshold.py +++ b/docs/sphinx_api_docs_source/public_api_missing_threshold.py @@ -4,166 +4,780 @@ adding an exclude directive to docs/sphinx_api_docs_source/public_api_excludes.py """ +import pathlib + +from docs.sphinx_api_docs_source.printable_definition import PrintableDefinition + ITEMS_IGNORED_FROM_PUBLIC_API = [ - "File: great_expectations/_docs_decorators.py Name: add", - "File: great_expectations/checkpoint/actions.py Name: _run", - "File: great_expectations/checkpoint/actions.py Name: get", - "File: great_expectations/checkpoint/actions.py Name: run", - "File: great_expectations/checkpoint/actions.py Name: update", - "File: great_expectations/checkpoint/checkpoint.py Name: describe_dict", - "File: great_expectations/compatibility/not_imported.py Name: is_version_greater_or_equal", - "File: great_expectations/compatibility/typing_extensions.py Name: override", - "File: great_expectations/core/batch.py Name: head", - "File: great_expectations/core/batch_definition.py Name: build_batch_request", - "File: great_expectations/core/expectation_diagnostics/expectation_doctor.py Name: print_diagnostic_checklist", - "File: great_expectations/core/expectation_diagnostics/expectation_doctor.py Name: run_diagnostics", - "File: great_expectations/core/expectation_suite.py Name: add_expectation_configuration", - "File: great_expectations/core/expectation_suite.py Name: remove_expectation", - "File: great_expectations/core/expectation_validation_result.py Name: describe_dict", - "File: great_expectations/core/factory/factory.py Name: add", - "File: great_expectations/core/factory/factory.py Name: all", - "File: great_expectations/core/factory/factory.py Name: get", - "File: great_expectations/core/metric_domain_types.py Name: MetricDomainTypes", - "File: great_expectations/core/metric_function_types.py Name: MetricPartialFunctionTypes", - "File: great_expectations/core/partitioners.py Name: ColumnPartitionerMonthly", - "File: great_expectations/core/partitioners.py Name: PartitionerColumnValue", - "File: great_expectations/core/yaml_handler.py Name: YAMLHandler", - "File: great_expectations/core/yaml_handler.py Name: dump", - "File: great_expectations/core/yaml_handler.py Name: load", - "File: great_expectations/data_context/data_context/abstract_data_context.py Name: add_store", - "File: great_expectations/data_context/data_context/abstract_data_context.py Name: delete_datasource", - "File: great_expectations/data_context/data_context/abstract_data_context.py Name: get_docs_sites_urls", - "File: great_expectations/data_context/data_context/abstract_data_context.py Name: get_validator", - "File: great_expectations/data_context/data_context/abstract_data_context.py Name: list_datasources", - "File: great_expectations/data_context/data_context/abstract_data_context.py Name: open_data_docs", - "File: great_expectations/data_context/data_context/context_factory.py Name: build_data_docs", - "File: great_expectations/data_context/data_context/context_factory.py Name: get_context", - "File: great_expectations/data_context/data_context/context_factory.py Name: get_docs_sites_urls", - "File: great_expectations/data_context/data_context/context_factory.py Name: get_validator", - "File: great_expectations/data_context/data_context_variables.py Name: save", - "File: great_expectations/data_context/store/_store_backend.py Name: add", - "File: great_expectations/data_context/store/_store_backend.py Name: update", - "File: great_expectations/data_context/store/checkpoint_store.py Name: CheckpointStore", - "File: great_expectations/data_context/store/database_store_backend.py Name: DatabaseStoreBackend", - "File: great_expectations/data_context/store/expectations_store.py Name: ExpectationsStore", - "File: great_expectations/data_context/store/metric_store.py Name: MetricStore", - "File: great_expectations/data_context/store/query_store.py Name: SqlAlchemyQueryStore", - "File: great_expectations/data_context/store/store.py Name: add", - "File: great_expectations/data_context/store/store.py Name: update", - "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleAzureBlobStoreBackend", - "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleFilesystemStoreBackend", - "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleGCSStoreBackend", - "File: great_expectations/data_context/store/tuple_store_backend.py Name: TupleS3StoreBackend", - "File: great_expectations/data_context/store/validation_definition_store.py Name: ValidationDefinitionStore", - "File: great_expectations/data_context/store/validation_results_store.py Name: ValidationResultsStore", - "File: great_expectations/data_context/types/base.py Name: update", - "File: great_expectations/data_context/types/resource_identifiers.py Name: GXCloudIdentifier", - "File: great_expectations/datasource/datasource_dict.py Name: add_dataframe_asset", - "File: great_expectations/datasource/fluent/config.py Name: yaml", - "File: great_expectations/datasource/fluent/config_str.py Name: ConfigStr", - "File: great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py Name: columns", - "File: great_expectations/datasource/fluent/data_asset/path/directory_asset.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/data_asset/path/directory_asset.py Name: get_batch_parameters_keys", - "File: great_expectations/datasource/fluent/data_asset/path/file_asset.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/data_asset/path/file_asset.py Name: get_batch_parameters_keys", - "File: great_expectations/datasource/fluent/data_asset/path/path_data_asset.py Name: get_batch", - "File: great_expectations/datasource/fluent/data_asset/path/path_data_asset.py Name: get_batch_parameters_keys", - "File: great_expectations/datasource/fluent/data_connector/batch_filter.py Name: validate", - "File: great_expectations/datasource/fluent/fabric.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/fabric.py Name: get_batch", - "File: great_expectations/datasource/fluent/fluent_base_model.py Name: yaml", - "File: great_expectations/datasource/fluent/invalid_datasource.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/invalid_datasource.py Name: get_asset", - "File: great_expectations/datasource/fluent/invalid_datasource.py Name: get_batch", - "File: great_expectations/datasource/fluent/invalid_datasource.py Name: get_batch_parameters_keys", - "File: great_expectations/datasource/fluent/pandas_datasource.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/pandas_datasource.py Name: get_batch", - "File: great_expectations/datasource/fluent/sources.py Name: delete_datasource", - "File: great_expectations/datasource/fluent/spark_datasource.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/spark_datasource.py Name: get_batch", - "File: great_expectations/datasource/fluent/sql_datasource.py Name: build_batch_request", - "File: great_expectations/datasource/fluent/sql_datasource.py Name: get_batch", - "File: great_expectations/exceptions/exceptions.py Name: DataContextError", - "File: great_expectations/exceptions/exceptions.py Name: InvalidExpectationConfigurationError", - "File: great_expectations/execution_engine/execution_engine.py Name: ExecutionEngine", - "File: great_expectations/execution_engine/execution_engine.py Name: get_compute_domain", - "File: great_expectations/execution_engine/pandas_execution_engine.py Name: PandasExecutionEngine", - "File: great_expectations/execution_engine/pandas_execution_engine.py Name: get_compute_domain", - "File: great_expectations/execution_engine/sparkdf_execution_engine.py Name: SparkDFExecutionEngine", - "File: great_expectations/execution_engine/sparkdf_execution_engine.py Name: get_compute_domain", - "File: great_expectations/execution_engine/sqlalchemy_execution_engine.py Name: SqlAlchemyExecutionEngine", - "File: great_expectations/execution_engine/sqlalchemy_execution_engine.py Name: execute_query", - "File: great_expectations/execution_engine/sqlalchemy_execution_engine.py Name: get_compute_domain", - "File: great_expectations/expectations/core/expect_column_max_to_be_between.py Name: ExpectColumnMaxToBeBetween", - "File: great_expectations/expectations/core/expect_column_to_exist.py Name: ExpectColumnToExist", - "File: great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py Name: ExpectColumnValuesToBeInTypeList", - "File: great_expectations/expectations/core/expect_column_values_to_be_null.py Name: ExpectColumnValuesToBeNull", - "File: great_expectations/expectations/core/expect_column_values_to_be_of_type.py Name: ExpectColumnValuesToBeOfType", - "File: great_expectations/expectations/core/expect_table_column_count_to_be_between.py Name: ExpectTableColumnCountToBeBetween", - "File: great_expectations/expectations/core/expect_table_column_count_to_equal.py Name: ExpectTableColumnCountToEqual", - "File: great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py Name: ExpectTableColumnsToMatchOrderedList", - "File: great_expectations/expectations/core/expect_table_columns_to_match_set.py Name: ExpectTableColumnsToMatchSet", - "File: great_expectations/expectations/core/expect_table_row_count_to_be_between.py Name: ExpectTableRowCountToBeBetween", - "File: great_expectations/expectations/core/expect_table_row_count_to_equal.py Name: ExpectTableRowCountToEqual", - "File: great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py Name: ExpectTableRowCountToEqualOtherTable", - "File: great_expectations/expectations/core/unexpected_rows_expectation.py Name: UnexpectedRowsExpectation", - "File: great_expectations/expectations/expectation.py Name: ColumnAggregateExpectation", - "File: great_expectations/expectations/expectation.py Name: ColumnMapExpectation", - "File: great_expectations/expectations/expectation.py Name: UnexpectedRowsExpectation", - "File: great_expectations/expectations/expectation.py Name: render_suite_parameter_string", - "File: great_expectations/expectations/expectation.py Name: validate_configuration", - "File: great_expectations/expectations/expectation_configuration.py Name: ExpectationConfiguration", - "File: great_expectations/expectations/expectation_configuration.py Name: to_domain_obj", - "File: great_expectations/expectations/expectation_configuration.py Name: type", - "File: great_expectations/expectations/metrics/column_aggregate_metric_provider.py Name: ColumnAggregateMetricProvider", - "File: great_expectations/expectations/metrics/column_aggregate_metric_provider.py Name: column_aggregate_partial", - "File: great_expectations/expectations/metrics/column_aggregate_metric_provider.py Name: column_aggregate_value", - "File: great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py Name: column_condition_partial", - "File: great_expectations/expectations/metrics/map_metric_provider/column_map_metric_provider.py Name: ColumnMapMetricProvider", - "File: great_expectations/expectations/metrics/metric_provider.py Name: MetricProvider", - "File: great_expectations/expectations/metrics/metric_provider.py Name: metric_partial", - "File: great_expectations/expectations/metrics/metric_provider.py Name: metric_value", - "File: great_expectations/expectations/regex_based_column_map_expectation.py Name: validate_configuration", - "File: great_expectations/expectations/set_based_column_map_expectation.py Name: validate_configuration", - "File: great_expectations/experimental/metric_repository/metric_retriever.py Name: get_validator", - "File: great_expectations/experimental/rule_based_profiler/helpers/util.py Name: build_batch_request", - "File: great_expectations/experimental/rule_based_profiler/rule_based_profiler.py Name: run", - "File: great_expectations/render/components.py Name: CollapseContent", - "File: great_expectations/render/components.py Name: RenderedStringTemplateContent", - "File: great_expectations/render/components.py Name: RenderedTableContent", - "File: great_expectations/render/components.py Name: validate", - "File: great_expectations/render/renderer/email_renderer.py Name: EmailRenderer", - "File: great_expectations/render/renderer/microsoft_teams_renderer.py Name: MicrosoftTeamsRenderer", - "File: great_expectations/render/renderer/opsgenie_renderer.py Name: OpsgenieRenderer", - "File: great_expectations/render/renderer/renderer.py Name: renderer", - "File: great_expectations/render/renderer/site_builder.py Name: DefaultSiteIndexBuilder", - "File: great_expectations/render/renderer/site_builder.py Name: SiteBuilder", - "File: great_expectations/render/renderer/slack_renderer.py Name: SlackRenderer", - "File: great_expectations/render/util.py Name: handle_strict_min_max", - "File: great_expectations/render/util.py Name: num_to_str", - "File: great_expectations/render/util.py Name: parse_row_condition_string_pandas_engine", - "File: great_expectations/render/util.py Name: substitute_none_for_missing", - "File: great_expectations/validator/metric_configuration.py Name: MetricConfiguration", - "File: great_expectations/validator/metrics_calculator.py Name: columns", - "File: great_expectations/validator/validation_graph.py Name: resolve", - "File: great_expectations/validator/validator.py Name: columns", - "File: great_expectations/validator/validator.py Name: head", - "File: great_expectations/validator/validator.py Name: remove_expectation", - "File: great_expectations/validator/validator.py Name: save_expectation_suite", - "File: great_expectations/validator/validator.py Name: validate", - # Expectations referenced in the Learn data quality use cases: - "File: great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py Name: ExpectColumnKLDivergenceToBeLessThan", - "File: great_expectations/expectations/core/expect_column_mean_to_be_between.py Name: ExpectColumnMeanToBeBetween", - "File: great_expectations/expectations/core/expect_column_median_to_be_between.py Name: ExpectColumnMedianToBeBetween", - "File: great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py Name: ExpectColumnQuantileValuesToBeBetween", - "File: great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py Name: ExpectColumnValueZScoresToBeLessThan", - "File: great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py Name: ExpectColumnPairValuesAToBeGreaterThanB", - "File: great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py Name: ExpectColumnPairValuesToBeEqual", - "File: great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py Name: ExpectMulticolumnSumToEqual", - "File: great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py Name: ExpectColumnProportionOfUniqueValuesToBeBetween", - "File: great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py Name: ExpectColumnUniqueValueCountToBeBetween", - "File: great_expectations/expectations/core/expect_column_values_to_be_unique.py Name: ExpectColumnValuesToBeUnique", - "File: great_expectations/expectations/core/expect_compound_columns_to_be_unique.py Name: ExpectCompoundColumnsToBeUnique", - "File: great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py Name: ExpectSelectColumnValuesToBeUniqueWithinRecord", - "File: great_expectations/expectations/core/expect_column_min_to_be_between.py Name: ExpectColumnMinToBeBetween", + PrintableDefinition( + file=pathlib.Path("great_expectations/_docs_decorators.py"), name="add" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/checkpoint/actions.py"), name="get" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/checkpoint/actions.py"), name="_run" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/checkpoint/actions.py"), name="run" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/checkpoint/actions.py"), name="update" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/checkpoint/checkpoint.py"), + name="describe_dict", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/compatibility/not_imported.py"), + name="is_version_greater_or_equal", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/compatibility/typing_extensions.py"), + name="override", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/batch.py"), name="head" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/batch_definition.py"), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/core/expectation_diagnostics/expectation_doctor.py" + ), + name="print_diagnostic_checklist", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/core/expectation_diagnostics/expectation_doctor.py" + ), + name="run_diagnostics", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/expectation_suite.py"), + name="add_expectation_configuration", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/expectation_suite.py"), + name="remove_expectation", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/expectation_validation_result.py"), + name="describe_dict", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/factory/factory.py"), name="add" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/factory/factory.py"), name="all" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/factory/factory.py"), name="get" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/metric_domain_types.py"), + name="MetricDomainTypes", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/metric_function_types.py"), + name="MetricPartialFunctionTypes", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/partitioners.py"), + name="ColumnPartitionerMonthly", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/partitioners.py"), + name="PartitionerColumnValue", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/yaml_handler.py"), name="YAMLHandler" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/yaml_handler.py"), name="dump" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/core/yaml_handler.py"), name="load" + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/abstract_data_context.py" + ), + name="add_store", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/abstract_data_context.py" + ), + name="delete_datasource", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/abstract_data_context.py" + ), + name="get_docs_sites_urls", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/abstract_data_context.py" + ), + name="get_validator", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/abstract_data_context.py" + ), + name="list_datasources", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/abstract_data_context.py" + ), + name="open_data_docs", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/context_factory.py" + ), + name="build_data_docs", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/context_factory.py" + ), + name="get_context", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/context_factory.py" + ), + name="get_docs_sites_urls", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/data_context/context_factory.py" + ), + name="get_validator", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/data_context_variables.py"), + name="save", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/_store_backend.py"), + name="add", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/_store_backend.py"), + name="update", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/checkpoint_store.py"), + name="CheckpointStore", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/database_store_backend.py" + ), + name="DatabaseStoreBackend", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/expectations_store.py" + ), + name="ExpectationsStore", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/metric_store.py"), + name="MetricStore", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/query_store.py"), + name="SqlAlchemyQueryStore", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/store.py"), name="add" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/store/store.py"), + name="update", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/tuple_store_backend.py" + ), + name="TupleAzureBlobStoreBackend", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/tuple_store_backend.py" + ), + name="TupleFilesystemStoreBackend", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/tuple_store_backend.py" + ), + name="TupleGCSStoreBackend", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/tuple_store_backend.py" + ), + name="TupleS3StoreBackend", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/validation_definition_store.py" + ), + name="ValidationDefinitionStore", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/store/validation_results_store.py" + ), + name="ValidationResultsStore", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/data_context/types/base.py"), + name="update", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/data_context/types/resource_identifiers.py" + ), + name="GXCloudIdentifier", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/config.py"), name="yaml" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/config_str.py"), + name="ConfigStr", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py" + ), + name="columns", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/directory_asset.py" + ), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/directory_asset.py" + ), + name="get_batch_parameters_keys", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/file_asset.py" + ), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/file_asset.py" + ), + name="get_batch_parameters_keys", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/path_data_asset.py" + ), + name="get_batch", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_asset/path/path_data_asset.py" + ), + name="get_batch_parameters_keys", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/datasource/fluent/data_connector/batch_filter.py" + ), + name="validate", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/fabric.py"), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/fabric.py"), + name="get_batch", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/fluent_base_model.py"), + name="yaml", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/invalid_datasource.py"), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/invalid_datasource.py"), + name="get_asset", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/invalid_datasource.py"), + name="get_batch", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/invalid_datasource.py"), + name="get_batch_parameters_keys", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/pandas_datasource.py"), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/pandas_datasource.py"), + name="get_batch", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/sources.py"), + name="delete_datasource", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/spark_datasource.py"), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/spark_datasource.py"), + name="get_batch", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/sql_datasource.py"), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/datasource/fluent/sql_datasource.py"), + name="get_batch", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/exceptions/exceptions.py"), + name="DataContextError", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/exceptions/exceptions.py"), + name="InvalidExpectationConfigurationError", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/execution_engine/execution_engine.py"), + name="ExecutionEngine", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/execution_engine/execution_engine.py"), + name="get_compute_domain", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/pandas_execution_engine.py" + ), + name="PandasExecutionEngine", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/pandas_execution_engine.py" + ), + name="get_compute_domain", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/sparkdf_execution_engine.py" + ), + name="SparkDFExecutionEngine", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/sparkdf_execution_engine.py" + ), + name="get_compute_domain", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/sqlalchemy_execution_engine.py" + ), + name="SqlAlchemyExecutionEngine", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/sqlalchemy_execution_engine.py" + ), + name="execute_query", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/execution_engine/sqlalchemy_execution_engine.py" + ), + name="get_compute_domain", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py" + ), + name="ExpectColumnKLDivergenceToBeLessThan", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_max_to_be_between.py" + ), + name="ExpectColumnMaxToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_mean_to_be_between.py" + ), + name="ExpectColumnMeanToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_median_to_be_between.py" + ), + name="ExpectColumnMedianToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_min_to_be_between.py" + ), + name="ExpectColumnMinToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py" + ), + name="ExpectColumnPairValuesAToBeGreaterThanB", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py" + ), + name="ExpectColumnPairValuesToBeEqual", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py" + ), + name="ExpectColumnProportionOfUniqueValuesToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py" + ), + name="ExpectColumnQuantileValuesToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_to_exist.py" + ), + name="ExpectColumnToExist", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py" + ), + name="ExpectColumnUniqueValueCountToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py" + ), + name="ExpectColumnValueZScoresToBeLessThan", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py" + ), + name="ExpectColumnValuesToBeInTypeList", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_values_to_be_null.py" + ), + name="ExpectColumnValuesToBeNull", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_values_to_be_of_type.py" + ), + name="ExpectColumnValuesToBeOfType", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_column_values_to_be_unique.py" + ), + name="ExpectColumnValuesToBeUnique", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_compound_columns_to_be_unique.py" + ), + name="ExpectCompoundColumnsToBeUnique", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py" + ), + name="ExpectMulticolumnSumToEqual", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py" + ), + name="ExpectSelectColumnValuesToBeUniqueWithinRecord", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_column_count_to_be_between.py" + ), + name="ExpectTableColumnCountToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_column_count_to_equal.py" + ), + name="ExpectTableColumnCountToEqual", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py" + ), + name="ExpectTableColumnsToMatchOrderedList", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_columns_to_match_set.py" + ), + name="ExpectTableColumnsToMatchSet", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_row_count_to_be_between.py" + ), + name="ExpectTableRowCountToBeBetween", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_row_count_to_equal.py" + ), + name="ExpectTableRowCountToEqual", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py" + ), + name="ExpectTableRowCountToEqualOtherTable", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/core/unexpected_rows_expectation.py" + ), + name="UnexpectedRowsExpectation", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/expectation.py"), + name="ColumnAggregateExpectation", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/expectation.py"), + name="ColumnMapExpectation", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/expectation.py"), + name="UnexpectedRowsExpectation", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/expectation.py"), + name="render_suite_parameter_string", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/expectation.py"), + name="validate_configuration", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/expectation_configuration.py" + ), + name="ExpectationConfiguration", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/expectation_configuration.py" + ), + name="to_domain_obj", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/expectation_configuration.py" + ), + name="type", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/metrics/column_aggregate_metric_provider.py" + ), + name="ColumnAggregateMetricProvider", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/metrics/column_aggregate_metric_provider.py" + ), + name="column_aggregate_partial", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/metrics/column_aggregate_metric_provider.py" + ), + name="column_aggregate_value", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py" + ), + name="column_condition_partial", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/metrics/map_metric_provider/column_map_metric_provider.py" + ), + name="ColumnMapMetricProvider", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/metrics/metric_provider.py"), + name="MetricProvider", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/metrics/metric_provider.py"), + name="metric_partial", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/expectations/metrics/metric_provider.py"), + name="metric_value", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/regex_based_column_map_expectation.py" + ), + name="validate_configuration", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/expectations/set_based_column_map_expectation.py" + ), + name="validate_configuration", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/experimental/metric_repository/metric_retriever.py" + ), + name="get_validator", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/experimental/rule_based_profiler/helpers/util.py" + ), + name="build_batch_request", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/experimental/rule_based_profiler/rule_based_profiler.py" + ), + name="run", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/components.py"), + name="CollapseContent", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/components.py"), + name="RenderedStringTemplateContent", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/components.py"), + name="RenderedTableContent", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/components.py"), name="validate" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/renderer/email_renderer.py"), + name="EmailRenderer", + ), + PrintableDefinition( + file=pathlib.Path( + "great_expectations/render/renderer/microsoft_teams_renderer.py" + ), + name="MicrosoftTeamsRenderer", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/renderer/opsgenie_renderer.py"), + name="OpsgenieRenderer", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/renderer/renderer.py"), + name="renderer", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/renderer/site_builder.py"), + name="DefaultSiteIndexBuilder", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/renderer/site_builder.py"), + name="SiteBuilder", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/renderer/slack_renderer.py"), + name="SlackRenderer", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/util.py"), + name="handle_strict_min_max", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/util.py"), name="num_to_str" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/util.py"), + name="parse_row_condition_string_pandas_engine", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/render/util.py"), + name="substitute_none_for_missing", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/metric_configuration.py"), + name="MetricConfiguration", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/metrics_calculator.py"), + name="columns", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/validation_graph.py"), + name="resolve", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/validator.py"), name="columns" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/validator.py"), name="head" + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/validator.py"), + name="remove_expectation", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/validator.py"), + name="save_expectation_suite", + ), + PrintableDefinition( + file=pathlib.Path("great_expectations/validator/validator.py"), name="validate" + ), ] diff --git a/docs/sphinx_api_docs_source/public_api_report.py b/docs/sphinx_api_docs_source/public_api_report.py index 9e7c040f29f5..5e9ba4bbb347 100755 --- a/docs/sphinx_api_docs_source/public_api_report.py +++ b/docs/sphinx_api_docs_source/public_api_report.py @@ -66,6 +66,7 @@ public_api_includes, public_api_missing_threshold, ) +from docs.sphinx_api_docs_source.printable_definition import PrintableDefinition if TYPE_CHECKING: from docs.sphinx_api_docs_source.include_exclude_definition import ( @@ -790,11 +791,11 @@ def write_printable_definitions_to_file( """ printable_definitions = self.generate_printable_definitions() with open(filepath, "w") as f: - f.write("\n".join(printable_definitions)) + f.write("\n".join([str(d) for d in printable_definitions])) def generate_printable_definitions( self, - ) -> List[str]: + ) -> List[PrintableDefinition]: """Generate a printable (human readable) definition. Returns: @@ -803,30 +804,30 @@ def generate_printable_definitions( sorted_definitions_list = sorted( list(self.definitions), key=operator.attrgetter("filepath", "name") ) - sorted_definitions_strings: List[str] = [] + sorted_printable_definitions: List[PrintableDefinition] = [] for definition in sorted_definitions_list: if definition.filepath.is_absolute(): - filepath = str(definition.filepath.relative_to(self.repo_root)) + filepath = definition.filepath.relative_to(self.repo_root) else: - filepath = str(definition.filepath) - sorted_definitions_strings.append( - f"File: {filepath} Name: {definition.name}" - ) + filepath = definition.filepath - sorted_definitions_strings_no_dupes = self._deduplicate_strings( - sorted_definitions_strings - ) + printable_definition = PrintableDefinition( + file=filepath, name=definition.name + ) + sorted_printable_definitions.append(printable_definition) - return sorted_definitions_strings_no_dupes + return self._deduplicate_definitions(sorted_printable_definitions) - def _deduplicate_strings(self, strings: List[str]) -> List[str]: + def _deduplicate_definitions( + self, printable_definitions: List[PrintableDefinition] + ) -> List[PrintableDefinition]: """Deduplicate a list of strings, keeping order intact.""" seen = set() no_duplicates = [] - for s in strings: - if s not in seen: - no_duplicates.append(s) - seen.add(s) + for definition in printable_definitions: + if definition not in seen: + no_duplicates.append(definition) + seen.add(definition) return no_duplicates @@ -934,14 +935,14 @@ def generate_public_api_report(write_to_file: bool = False) -> None: f"Items are missing from the public API: {len(undocumented_and_unignored)}" ) for item in sorted(undocumented_and_unignored): - logger.error(" - " + item) + logger.error(" - " + str(item)) has_errors = True if documented_and_ignored: logger.error( f"Items that should be removed from public_api_missing_threshold.ITEMS_IGNORED_FROM_PUBLIC_API: {len(documented_and_ignored)}" ) for item in sorted(documented_and_ignored): - logger.error(" - " + item) + logger.error(" - " + str(item)) has_errors = True if has_errors: diff --git a/tests/scripts/test_public_api_report.py b/tests/scripts/test_public_api_report.py index b254e40d6da9..2b0d163df89a 100644 --- a/tests/scripts/test_public_api_report.py +++ b/tests/scripts/test_public_api_report.py @@ -150,27 +150,31 @@ def sample_markdown_doc_with_yaml() -> str: @pytest.fixture -def repo_root() -> pathlib.Path: - return pathlib.Path("/some/absolute/path/repo_root/") +def repo_root(tmp_path) -> pathlib.Path: + return tmp_path @pytest.fixture def sample_docs_example_python_file_string_filepath( repo_root: pathlib.Path, ) -> pathlib.Path: - return ( + path = ( repo_root / pathlib.Path("tests/integration/docusaurus/sample_docs_example_python_file_string.py") ).relative_to(repo_root) + path.touch() + return path @pytest.fixture def sample_with_definitions_python_file_string_filepath( repo_root: pathlib.Path, ) -> pathlib.Path: - return ( - repo_root / pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") - ).relative_to(repo_root) + path = (repo_root / pathlib.Path("sample_with_definitions_python_file_string.py")).relative_to( + repo_root + ) + path.touch() + return path @pytest.fixture @@ -222,9 +226,6 @@ def empty_docs_example_parser( class TestDocExampleParser: - def test_instantiate(self, docs_example_parser: DocsExampleParser): - assert isinstance(docs_example_parser, DocsExampleParser) - def test_retrieve_all_usages_in_files(self, docs_example_parser: DocsExampleParser): usages = docs_example_parser.get_names_from_usage_in_docs_examples() assert usages == { @@ -251,9 +252,6 @@ def code_parser(sample_with_definitions_file_contents: FileContents) -> CodePars class TestCodeParser: - def test_instantiate(self, code_parser: CodeParser): - assert isinstance(code_parser, CodeParser) - def test_get_all_class_method_and_function_names(self, code_parser: CodeParser): names = code_parser.get_all_class_method_and_function_names() assert names == { @@ -298,7 +296,7 @@ def test_get_all_class_method_and_function_definitions(self, code_parser: CodePa "example_staticmethod", } assert {d.filepath for d in definitions} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } @@ -372,16 +370,11 @@ def test__get_import_names(various_imports: str): @pytest.fixture -def public_api_checker( - docs_example_parser: DocsExampleParser, code_parser: CodeParser -) -> PublicAPIChecker: +def public_api_checker(code_parser: CodeParser) -> PublicAPIChecker: return PublicAPIChecker(code_parser=code_parser) class TestPublicAPIChecker: - def test_instantiate(self, public_api_checker: PublicAPIChecker): - assert isinstance(public_api_checker, PublicAPIChecker) - def test_get_all_public_api_definitions(self, public_api_checker: PublicAPIChecker): observed = public_api_checker.get_all_public_api_definitions() assert len(observed) == 6 @@ -394,7 +387,7 @@ def test_get_all_public_api_definitions(self, public_api_checker: PublicAPICheck "example_public_staticmethod", } assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } def _class_and_function_definitions( @@ -408,7 +401,9 @@ def _class_and_function_definitions( return definitions - def test_is_definition_marked_public_api_yes(self, public_api_checker: PublicAPIChecker): + def test_is_definition_marked_public_api_yes( + self, public_api_checker: PublicAPIChecker, tmp_path: pathlib.Path + ): file_string = """ @public_api def example_public_api_module_level_function(): @@ -437,11 +432,12 @@ def example_multiple_decorator_public_api_method(self): pass """ + test_path = tmp_path / "test_path.py" ast_definitions = self._class_and_function_definitions(tree=ast.parse(file_string)) definitions = [ Definition( name="test_name", - filepath=pathlib.Path("test_path"), + filepath=pathlib.Path(test_path), ast_definition=ast_definition, ) for ast_definition in ast_definitions @@ -451,7 +447,9 @@ def example_multiple_decorator_public_api_method(self): for definition in definitions ) - def test_is_definition_marked_public_api_no(self, public_api_checker: PublicAPIChecker): + def test_is_definition_marked_public_api_no( + self, public_api_checker: PublicAPIChecker, tmp_path: pathlib.Path + ): file_string = """ def example_module_level_function(): pass @@ -474,11 +472,12 @@ def example_multiple_decorator_public_method(self): pass """ + test_path = tmp_path / "test_path.py" ast_definitions = self._class_and_function_definitions(tree=ast.parse(file_string)) definitions = [ Definition( name="test_name", - filepath=pathlib.Path("test_path"), + filepath=pathlib.Path(test_path), ast_definition=ast_definition, ) for ast_definition in ast_definitions @@ -510,6 +509,7 @@ def code_reference_filter_with_non_default_include_exclude( docs_example_parser: DocsExampleParser, code_parser: CodeParser, public_api_checker: PublicAPIChecker, + sample_docs_example_python_file_string_filepath: pathlib.Path, ) -> CodeReferenceFilter: return CodeReferenceFilter( repo_root=repo_root, @@ -518,12 +518,16 @@ def code_reference_filter_with_non_default_include_exclude( public_api_checker=public_api_checker, includes=[ IncludeExcludeDefinition( - reason="test", name="test_name", filepath=pathlib.Path("test_path") + reason="test", + name="test_name", + filepath=sample_docs_example_python_file_string_filepath, ) ], excludes=[ IncludeExcludeDefinition( - reason="test", name="test_name", filepath=pathlib.Path("test_path") + reason="test", + name="test_name", + filepath=sample_docs_example_python_file_string_filepath, ) ], ) @@ -552,6 +556,7 @@ def code_reference_filter_with_exclude_by_file( docs_example_parser: DocsExampleParser, code_parser: CodeParser, public_api_checker: PublicAPIChecker, + sample_with_definitions_python_file_string_filepath: pathlib.Path, ) -> CodeReferenceFilter: return CodeReferenceFilter( repo_root=repo_root, @@ -562,9 +567,7 @@ def code_reference_filter_with_exclude_by_file( excludes=[ IncludeExcludeDefinition( reason="test", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ) ], ) @@ -592,6 +595,7 @@ def code_reference_filter_with_exclude_by_file_and_name( docs_example_parser: DocsExampleParser, code_parser: CodeParser, public_api_checker: PublicAPIChecker, + sample_with_definitions_python_file_string_filepath: pathlib.Path, ) -> CodeReferenceFilter: return CodeReferenceFilter( repo_root=repo_root, @@ -603,16 +607,12 @@ def code_reference_filter_with_exclude_by_file_and_name( IncludeExcludeDefinition( reason="test", name="example_method", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), IncludeExcludeDefinition( reason="test", name="example_module_level_function", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), ], ) @@ -624,6 +624,7 @@ def code_reference_filter_with_include_by_file_and_name_already_included( docs_example_parser: DocsExampleParser, code_parser: CodeParser, public_api_checker: PublicAPIChecker, + sample_with_definitions_python_file_string_filepath: pathlib.Path, ) -> CodeReferenceFilter: return CodeReferenceFilter( repo_root=repo_root, @@ -634,16 +635,12 @@ def code_reference_filter_with_include_by_file_and_name_already_included( IncludeExcludeDefinition( reason="test", name="example_method", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), IncludeExcludeDefinition( reason="test", name="example_module_level_function", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), ], excludes=[], @@ -656,6 +653,7 @@ def code_reference_filter_with_include_by_file_and_name_already_excluded( docs_example_parser: DocsExampleParser, code_parser: CodeParser, public_api_checker: PublicAPIChecker, + sample_with_definitions_python_file_string_filepath: pathlib.Path, ) -> CodeReferenceFilter: return CodeReferenceFilter( repo_root=repo_root, @@ -666,24 +664,18 @@ def code_reference_filter_with_include_by_file_and_name_already_excluded( IncludeExcludeDefinition( reason="test", name="example_method", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), IncludeExcludeDefinition( reason="test", name="example_module_level_function", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), ], excludes=[ IncludeExcludeDefinition( reason="test", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ) ], ) @@ -695,6 +687,7 @@ def code_reference_filter_with_include_by_file_and_name_not_used_in_docs_example docs_example_parser: DocsExampleParser, code_parser: CodeParser, public_api_checker: PublicAPIChecker, + sample_with_definitions_python_file_string_filepath: pathlib.Path, ) -> CodeReferenceFilter: return CodeReferenceFilter( repo_root=repo_root, @@ -705,17 +698,13 @@ def code_reference_filter_with_include_by_file_and_name_not_used_in_docs_example IncludeExcludeDefinition( reason="test", name="example_no_usages_in_sample_docs_example_python_file_string", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ), ], excludes=[ IncludeExcludeDefinition( reason="test", - filepath=pathlib.Path( - "great_expectations/sample_with_definitions_python_file_string.py" - ), + filepath=sample_with_definitions_python_file_string_filepath, ) ], ) @@ -723,7 +712,6 @@ def code_reference_filter_with_include_by_file_and_name_not_used_in_docs_example class TestCodeReferenceFilter: def test_instantiate(self, code_reference_filter: CodeReferenceFilter): - assert isinstance(code_reference_filter, CodeReferenceFilter) assert code_reference_filter.excludes assert code_reference_filter.includes @@ -732,7 +720,6 @@ def test_instantiate_with_non_default_include_exclude( code_reference_filter_with_non_default_include_exclude: CodeReferenceFilter, ): code_reference_filter = code_reference_filter_with_non_default_include_exclude - assert isinstance(code_reference_filter, CodeReferenceFilter) assert code_reference_filter.excludes assert code_reference_filter.includes assert len(code_reference_filter.excludes) == 1 @@ -755,7 +742,7 @@ def test_filter_definitions_no_include_exclude( "example_staticmethod", } assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } def test_filter_definitions_with_references_from_docs_content( @@ -766,7 +753,7 @@ def test_filter_definitions_with_references_from_docs_content( assert len(observed) == 1 assert {d.name for d in observed} == {"ExampleClass"} assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } def test_filter_definitions_exclude_by_file( @@ -789,7 +776,7 @@ def test_filter_definitions_exclude_by_file_and_name( "example_staticmethod", } assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } def test_filter_definitions_include_by_file_and_name_already_included( @@ -815,7 +802,7 @@ def test_filter_definitions_include_by_file_and_name_already_included( "example_staticmethod", } assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } def test_filter_definitions_include_by_file_and_name_already_excluded( @@ -835,7 +822,7 @@ def test_filter_definitions_include_by_file_and_name_already_excluded( "example_module_level_function", } assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } def test_filter_definitions_include_by_file_and_name_already_excluded_not_used_in_docs_example( @@ -853,7 +840,7 @@ def test_filter_definitions_include_by_file_and_name_already_excluded_not_used_i "example_no_usages_in_sample_docs_example_python_file_string", } assert {d.filepath for d in observed} == { - pathlib.Path("great_expectations/sample_with_definitions_python_file_string.py") + pathlib.Path("sample_with_definitions_python_file_string.py") } @@ -880,25 +867,17 @@ def public_api_report_filter_out_file( class TestPublicAPIReport: - def test_instantiate(self, public_api_report: PublicAPIReport): - assert isinstance(public_api_report, PublicAPIReport) - def test_generate_printable_definitions(self, public_api_report: PublicAPIReport): expected: List[str] = [ - "File: great_expectations/sample_with_definitions_python_file_string.py Name: " - "ExampleClass", - "File: great_expectations/sample_with_definitions_python_file_string.py Name: " - "example_classmethod", - "File: great_expectations/sample_with_definitions_python_file_string.py Name: " - "example_method", - "File: great_expectations/sample_with_definitions_python_file_string.py Name: " - "example_method_with_args", - "File: great_expectations/sample_with_definitions_python_file_string.py Name: " + "File: sample_with_definitions_python_file_string.py Name: " "ExampleClass", + "File: sample_with_definitions_python_file_string.py Name: " "example_classmethod", + "File: sample_with_definitions_python_file_string.py Name: " "example_method", + "File: sample_with_definitions_python_file_string.py Name: " "example_method_with_args", + "File: sample_with_definitions_python_file_string.py Name: " "example_module_level_function", - "File: great_expectations/sample_with_definitions_python_file_string.py Name: " - "example_staticmethod", + "File: sample_with_definitions_python_file_string.py Name: " "example_staticmethod", ] - observed = public_api_report.generate_printable_definitions() + observed = [str(p) for p in public_api_report.generate_printable_definitions()] assert observed == expected def test_generate_printable_definitions_exclude_by_file( @@ -910,19 +889,19 @@ def test_generate_printable_definitions_exclude_by_file( class TestIncludeExcludeDefinition: - def test_instantiate_name_and_filepath(self): - definition = IncludeExcludeDefinition( - reason="reason", name="name", filepath=pathlib.Path("filepath") + def test_instantiate_name_and_filepath(self, tmp_path: pathlib.Path): + path = tmp_path / "test_path.py" + path.touch() + IncludeExcludeDefinition( + reason="reason", + name="name", + filepath=path, ) - assert isinstance(definition, IncludeExcludeDefinition) - - def test_instantiate_filepath_only(self): - definition = IncludeExcludeDefinition(reason="reason", filepath=pathlib.Path("filepath")) - assert isinstance(definition, IncludeExcludeDefinition) - def test_instantiate_name_and_filepath_no_reason(self): - with pytest.raises(TypeError): - IncludeExcludeDefinition(name="name", filepath=pathlib.Path("filepath")) + def test_instantiate_filepath_only(self, tmp_path: pathlib.Path): + path = tmp_path / "test_path.py" + path.touch() + IncludeExcludeDefinition(reason="reason", filepath=path) def test_instantiate_name_only(self): with pytest.raises(ValueError) as exc: