From 2d916aa316196aaced30c6d3ec7dd056cf7578a0 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 10 Jan 2023 16:48:43 -0500 Subject: [PATCH] [DOCS] Preparation for building api docs (#6737) --- docs/build_docs | 17 +++ .../build_sphinx_api_docs.py | 25 ++++- docusaurus.config.js | 8 +- .../data_context/abstract_data_context.py | 6 +- great_expectations/util.py | 103 ++++++++++++------ src/css/api_docs/basic-modified.scss | 4 +- 6 files changed, 119 insertions(+), 44 deletions(-) create mode 100755 docs/build_docs diff --git a/docs/build_docs b/docs/build_docs new file mode 100755 index 000000000000..3d47017a3a8a --- /dev/null +++ b/docs/build_docs @@ -0,0 +1,17 @@ +#!/bin/bash + +# Build API docs then build docusaurus docs. +# Currently used in our netlify pipeline. +# Note: Will be modified to build API docs on latest tagged release. + +echo "Installing dev dependencies" +pip install -c constraints-dev.txt -e ".[test]" + +echo "Installing dev dependencies" +(cd docs/sphinx_api_docs_source; pip install -r requirements-dev-api-docs.txt) + +echo "Building sphinx API docs." +(cd docs/sphinx_api_docs_source; invoke docs) + +echo "Building docusaurus docs." +yarn build diff --git a/docs/sphinx_api_docs_source/build_sphinx_api_docs.py b/docs/sphinx_api_docs_source/build_sphinx_api_docs.py index f02bb5cdb597..12cc98af1e14 100644 --- a/docs/sphinx_api_docs_source/build_sphinx_api_docs.py +++ b/docs/sphinx_api_docs_source/build_sphinx_api_docs.py @@ -208,11 +208,16 @@ def _build_class_md_stubs(self) -> None: for definition in definitions: if isinstance(definition.ast_definition, ast.ClassDef): md_stub = self._create_class_md_stub(definition=definition) - self._write_stub(stub=md_stub, definition=definition) + self._write_stub( + stub=md_stub, + file_name=self._get_md_file_name_from_entity_name( + definition=definition + ), + ) - def _write_stub(self, stub: str, definition: Definition) -> None: + def _write_stub(self, stub: str, file_name: str) -> None: """Write the markdown stub file with appropriate filename.""" - filepath = self.base_path / self._get_md_file_name(definition=definition) + filepath = self.base_path / file_name with filepath.open("a") as f: f.write(stub) @@ -223,17 +228,27 @@ def _build_module_md_stubs(self): for definition in definitions: md_stub = self._create_module_md_stub(definition=definition) - self._write_stub(stub=md_stub, definition=definition) + self._write_stub( + stub=md_stub, + file_name=self._get_md_file_name_from_dotted_path_prefix( + definition=definition + ), + ) def _get_entity_name(self, definition: Definition): """Get the name of the entity (class, module, function).""" return definition.ast_definition.name - def _get_md_file_name(self, definition: Definition): + def _get_md_file_name_from_entity_name(self, definition: Definition): """Generate markdown file name from the entity definition.""" snake_name = camel_to_snake(self._get_entity_name(definition=definition)) return f"{snake_name}.md" + def _get_md_file_name_from_dotted_path_prefix(self, definition: Definition): + """Generate markdown file name from the dotted path prefix.""" + dotted_path_prefix = self._get_dotted_path_prefix(definition=definition) + return f"{dotted_path_prefix}.md" + def _get_dotted_path_to_class(self, definition: Definition): """Get the dotted path to a class. diff --git a/docusaurus.config.js b/docusaurus.config.js index 644aa3fb30d4..e124d303ba9d 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -219,15 +219,15 @@ module.exports = { html: ` - + - + - + - + `, }, diff --git a/great_expectations/data_context/data_context/abstract_data_context.py b/great_expectations/data_context/data_context/abstract_data_context.py index ec148a7967f7..f92fee0becdd 100644 --- a/great_expectations/data_context/data_context/abstract_data_context.py +++ b/great_expectations/data_context/data_context/abstract_data_context.py @@ -164,12 +164,12 @@ T = TypeVar("T", dict, list, str) +@public_api class AbstractDataContext(ConfigPeer, ABC): - """ - Base class for all DataContexts that contain all context-agnostic data context operations. + """Base class for all Data Contexts that contains shared functionality. The class encapsulates most store / core components and convenience methods used to access them, meaning the - majority of DataContext functionality lives here. + majority of Data Context functionality lives here. """ # NOTE: These can become a property like ExpectationsStore.__name__ or placed in a separate diff --git a/great_expectations/util.py b/great_expectations/util.py index 05dc17923df5..3af485e9969b 100644 --- a/great_expectations/util.py +++ b/great_expectations/util.py @@ -55,6 +55,7 @@ from pkg_resources import Distribution from typing_extensions import Literal, TypeGuard +from great_expectations.core._docs_decorators import deprecated_argument, public_api from great_expectations.exceptions import ( GXCloudConfigurationError, PluginClassNotFoundError, @@ -1764,6 +1765,11 @@ def get_context( ... +@public_api +@deprecated_argument(argument_name="ge_cloud_base_url", version="0.15.37") +@deprecated_argument(argument_name="ge_cloud_access_token", version="0.15.37") +@deprecated_argument(argument_name="ge_cloud_organization_id", version="0.15.37") +@deprecated_argument(argument_name="ge_cloud_mode", version="0.15.37") def get_context( project_config: Optional[Union[DataContextConfig, Mapping]] = None, context_root_dir: Optional[PathStr] = None, @@ -1778,54 +1784,89 @@ def get_context( ge_cloud_organization_id: Optional[str] = None, ge_cloud_mode: Optional[bool] = None, ) -> AbstractDataContext: - """ - Method to return the appropriate DataContext depending on parameters and environment. + """Method to return the appropriate Data Context depending on parameters and environment. Usage: - import great_expectations as gx - my_context = gx.get_context([parameters]) + `import great_expectations as gx` + + `my_context = gx.get_context()` + + This method returns the appropriate Data Context based on which parameters you've passed and / or your environment configuration: + + - FileDataContext: Configuration stored in a file. + + - EphemeralDataContext: Configuration passed in at runtime. + + - CloudDataContext: Configuration stored in Great Expectations Cloud. + + Read on for more details about each of the Data Context types: + + **FileDataContext:** A Data Context configured via a yaml file. Returned by default if you have no cloud configuration set up and pass no parameters. If you pass context_root_dir, we will look for a great_expectations.yml configuration there. If not we will look at the following locations: + + - Path defined in a GX_HOME environment variable. + + - The current directory. + + - Parent directories of the current directory (e.g. in case you invoke the CLI in a sub folder of your Great Expectations directory). - 1. If gx.get_context() is run in a filesystem where `great_expectations init` has been run, then it will return a - FileDataContext. + Relevant parameters - 2. If gx.get_context() is passed in a `context_root_dir` (which contains great_expectations.yml) then it will return - a FileDataContext. + - context_root_dir: Provide an alternative directory to look for GX config. - 3. If gx.get_context() is passed in an in-memory `project_config` then it will return EphemeralDataContext. - `context_root_dir` can also be passed in, but the configurations from the in-memory config will override the - configurations in the `great_expectations.yml` file. + - project_config: Optionally override the configuration on disk - only if `context_root_dir` is also provided. + - runtime_environment: Optionally override specific configuration values. - 4. If GX is being run in the Cloud, and the information needed for gx_cloud_config (ie gx_cloud_base_url, - gx_cloud_access_token, gx_cloud_organization_id) are passed in as parameters to get_context(), configured as - environment variables, or in a .conf file, then get_context() will return a CloudDataContext. + **EphemeralDataContext:** A temporary, in-memory Data Context typically used in a pipeline. The default if you pass in only a project_config and have no cloud configuration set up. + Relevant parameters - +-----------------------+---------------------+---------------+ - | get_context params | Env Not Config'd | Env Config'd | - +-----------------------+---------------------+---------------+ - | () | Local | Cloud | - | (cloud_mode=True) | Exception! | Cloud | - | (cloud_mode=False) | Local | Local | - +-----------------------+---------------------+---------------+ + - project_config: Used to configure the Data Context. + + - runtime_environment: Optionally override specific configuration values. + + **CloudDataContext:** A Data Context whose configuration comes from Great Expectations Cloud. The default if you have a cloud configuration set up. Pass `cloud_mode=False` if you have a cloud configuration set up and you do not wish to create a CloudDataContext. + + Cloud configuration can be set up by passing `cloud_*` parameters to `get_context()`, configuring cloud environment variables, or in a great_expectations.conf file. + + Relevant parameters + + - cloud_base_url: Override env var or great_expectations.conf file. + + - cloud_access_token: Override env var or great_expectations.conf file. + + - cloud_organization_id: Override env var or great_expectations.conf file. + + - cloud_mode: Set to True or False to explicitly enable/disable cloud mode. + + - project_config: Optionally override the cloud configuration. + + - runtime_environment: Optionally override specific configuration values. Args: - project_config (dict or DataContextConfig): In-memory configuration for DataContext. + project_config: In-memory configuration for Data Context. context_root_dir (str or pathlib.Path): Path to directory that contains great_expectations.yml file - runtime_environment (dict): A dictionary of values can be passed to a DataContext when it is instantiated. + runtime_environment: A dictionary of values can be passed to a DataContext when it is instantiated. These values will override both values from the config variables file and from environment variables. - - The following parameters are relevant when running ge_cloud - * cloud_base_url (str): url for GX Cloud endpoint. - * cloud_access_token (str): access_token for GX Cloud account. - * cloud_organization_id (str): org_id for GX Cloud account. - * cloud_mode (bool): bool flag to specify whether to run GX in Cloud mode (default is None). + cloud_base_url: url for GX Cloud endpoint. + cloud_access_token: access_token for GX Cloud account. + cloud_organization_id: org_id for GX Cloud account. + cloud_mode: whether to run GX in Cloud mode (default is None). + If None, cloud mode is assumed if cloud credentials are set up. Set to False to override. + ge_cloud_base_url: url for GX Cloud endpoint. + ge_cloud_access_token: access_token for GX Cloud account. + ge_cloud_organization_id: org_id for GX Cloud account. + ge_cloud_mode: whether to run GX in Cloud mode (default is None). + If None, cloud mode is assumed if cloud credentials are set up. Set to False to override. Returns: - DataContext. Either a FileDataContext, EpehemralDataContext, or CloudDataContext depending on environment and/or - parameters + A Data Context. Either a FileDataContext, EphemeralDataContext, or + CloudDataContext depending on environment and/or + parameters. + Raises: + GXCloudConfigurationError: Cloud mode enabled, but missing configuration. """ from great_expectations.data_context.data_context import ( CloudDataContext, diff --git a/src/css/api_docs/basic-modified.scss b/src/css/api_docs/basic-modified.scss index 143733f29e22..b02969fc7914 100644 --- a/src/css/api_docs/basic-modified.scss +++ b/src/css/api_docs/basic-modified.scss @@ -621,7 +621,9 @@ dl.field-list > dt { } dl.field-list > dt:after { - content: ":"; + // Note: colon in content removed since it was duplicating + // on Netlify deploy. + content: ""; } dl.field-list > dd {