From 3e7a11a3af76676d5fcf88aa5fbf5109babc3d03 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Tue, 31 Oct 2023 15:50:40 -0400 Subject: [PATCH 01/10] view Co-authored-by: Bill Dirks Co-authored-by: Anthony Burdi Co-authored-by: T Pham Co-authored-by: Chetan Kini Co-authored-by: Nathan Farmer --- great_expectations/render/view/view.py | 35 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/great_expectations/render/view/view.py b/great_expectations/render/view/view.py index 0c40034e3c16..0fed08e9f55c 100644 --- a/great_expectations/render/view/view.py +++ b/great_expectations/render/view/view.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import datetime import json import re from collections import OrderedDict from string import Template as pTemplate +from typing import TYPE_CHECKING, ClassVar from uuid import uuid4 import mistune @@ -14,8 +17,10 @@ select_autoescape, ) +from great_expectations.compatibility.typing_extensions import override + try: - from jinja2 import contextfilter + from jinja2 import contextfilter # type: ignore[attr-defined] # for jinja 2.0 except ImportError: from jinja2 import pass_context as contextfilter @@ -26,6 +31,9 @@ RenderedDocumentContent, ) +if TYPE_CHECKING: + from jinja2 import BaseLoader, Template + class NoOpTemplate: def render(self, document): @@ -55,7 +63,7 @@ class DefaultJinjaView: """ - _template = NoOpTemplate + _template: ClassVar[str] def __init__( self, custom_styles_directory=None, custom_views_directory=None @@ -66,7 +74,7 @@ def __init__( templates_loader = PackageLoader("great_expectations", "render/view/templates") styles_loader = PackageLoader("great_expectations", "render/view/static/styles") - loaders = [templates_loader, styles_loader] + loaders: list[BaseLoader] = [templates_loader, styles_loader] if self.custom_styles_directory: loaders.append(FileSystemLoader(self.custom_styles_directory)) if self.custom_views_directory: @@ -109,11 +117,8 @@ def render(self, document, template=None, **kwargs): document = document.to_json_dict() return t.render(document, **kwargs) - def _get_template(self, template): - if template is None: - return NoOpTemplate - - template = self.env.get_template(template) + def _get_template(self, template_str: str) -> Template: + template = self.env.get_template(template_str) template.globals["now"] = lambda: datetime.datetime.now(datetime.timezone.utc) return template @@ -187,7 +192,7 @@ def render_content_block( # noqa: PLR0911, PLR0913, PLR0912 template_filename = f"markdown_{content_block_type}.j2" else: template_filename = f"{content_block_type}.j2" - template = self._get_template(template=template_filename) + template = self._get_template(template_str=template_filename) if content_block_id: return template.render( jinja_context, @@ -435,6 +440,7 @@ def _validate_document(self, document) -> None: class DefaultJinjaPageView(DefaultJinjaView): _template = "page.j2" + @override def _validate_document(self, document) -> None: assert isinstance(document, RenderedDocumentContent) @@ -446,6 +452,7 @@ class DefaultJinjaIndexPageView(DefaultJinjaPageView): class DefaultJinjaSectionView(DefaultJinjaView): _template = "section.j2" + @override def _validate_document(self, document) -> None: assert isinstance( document["section"], dict @@ -455,6 +462,7 @@ def _validate_document(self, document) -> None: class DefaultJinjaComponentView(DefaultJinjaView): _template = "component.j2" + @override def _validate_document(self, document) -> None: assert isinstance( document["content_block"], dict @@ -466,7 +474,8 @@ class DefaultMarkdownPageView(DefaultJinjaView): Convert a document to markdown format. """ - def _validate_document(self, document: RenderedDocumentContent) -> bool: + @override + def _validate_document(self, document: RenderedDocumentContent) -> None: """ Validate that the document is of the appropriate type at runtime. """ @@ -490,7 +499,8 @@ def render(self, document, template=None, **kwargs): else: return super().render(document=document, template=template, **kwargs) - def render_string_template(self, template: pTemplate) -> pTemplate: + @override + def render_string_template(self, template: pTemplate) -> pTemplate | str: """ Render string for markdown rendering. Bold all parameters and perform substitution. Args: @@ -547,10 +557,11 @@ def render_string_template(self, template: pTemplate) -> pTemplate: "$PARAMETER", "$$PARAMETER" ) - return pTemplate(template.get("template")).safe_substitute( + return pTemplate(template["template"]).safe_substitute( template.get("params", {}) ) + @override @contextfilter def render_content_block( # noqa: PLR0913 self, From 0774d12aa9ace34155295110e2d0afe6b97b9883 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Tue, 31 Oct 2023 15:52:12 -0400 Subject: [PATCH 02/10] view typing --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0ac35fdc6a4f..5c4cd73d1e66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,7 +146,6 @@ exclude = [ 'render/renderer/suite_scaffold_notebook_renderer\.py', # 7 'render/renderer/v3/suite_edit_notebook_renderer\.py', # 11 'render/renderer/v3/suite_profile_notebook_renderer\.py', # 4 - 'render/view/view\.py', # 11 'rule_based_profiler/domain_builder/map_metric_column_domain_builder\.py', # 8 'rule_based_profiler/estimators/bootstrap_numeric_range_estimator\.py', # 8 'rule_based_profiler/estimators/kde_numeric_range_estimator\.py', # 7 From b0489a016783a9f4d1e4fdcc73ddebfbcaafce16 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Tue, 31 Oct 2023 16:02:41 -0400 Subject: [PATCH 03/10] move legacy code to exclude section --- pyproject.toml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c4cd73d1e66..cc4272c5c243 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,18 +73,24 @@ exclude = [ 'cli/upgrade_helpers/upgrade_helper_v13\.py', # 17 - This is legacy code and will not be typed. 'dataset/sparkdf_dataset\.py', # 3 - This is legacy code and will not be typed. 'dataset/sqlalchemy_dataset\.py', # 16 - This is legacy code and will not be typed. + 'core/usage_statistics/anonymizers/batch_anonymizer\.py', # 10 - This code will be removed in 1.0 + 'core/usage_statistics/anonymizers/batch_request_anonymizer\.py', # 16 - This code will be removed in 1.0 + 'core/usage_statistics/anonymizers/checkpoint_anonymizer\.py', # 16 - This code will be removed in 1.0 + 'core/usage_statistics/anonymizers/data_docs_anonymizer\.py', # 5 - This code will be removed in 1.0 + 'core/usage_statistics/anonymizers/datasource_anonymizer\.py', # 9 - This code will be removed in 1.0 + 'core/usage_statistics/anonymizers/expectation_anonymizer\.py', # 6 - This code will be removed in 1.0 + 'core/usage_statistics/anonymizers/validation_operator_anonymizer\.py', # 5 - This code will be removed in 1.0 + 'render/renderer/v3/suite_edit_notebook_renderer\.py', # 11 - This is legacy code and will not be typed. + 'render/renderer/v3/suite_profile_notebook_renderer\.py', # 4 - This is legacy code and will not be typed. + 'render/renderer/suite_edit_notebook_renderer\.py', # 7 - This is legacy code and will not be typed. + 'render/renderer/suite_scaffold_notebook_renderer\.py', # 7 - This is legacy code and will not be typed. + 'render/renderer/datasource_new_notebook_renderer\.py', # 4 - This is legacy code and will not be typed. # END ALWAYS EXCLUDE SECTION ###################################################### # # ################################################################################# # TODO: complete typing for the following modules and remove from exclude list # number is the current number of typing errors for the excluded pattern - 'core/usage_statistics/anonymizers/batch_anonymizer\.py', # 10 - 'core/usage_statistics/anonymizers/batch_request_anonymizer\.py', # 16 - 'core/usage_statistics/anonymizers/checkpoint_anonymizer\.py', # 16 - 'core/usage_statistics/anonymizers/data_docs_anonymizer\.py', # 5 - 'core/usage_statistics/anonymizers/datasource_anonymizer\.py', # 9 - 'core/usage_statistics/anonymizers/expectation_anonymizer\.py', # 6 - 'core/usage_statistics/anonymizers/validation_operator_anonymizer\.py', # 5 + 'expectations/core/expect_column_values_to_be_of_type\.py', # 12 'expectations/core/expect_column_values_to_not_match_regex_list\.py', # 2 'expectations/core/expect_column_values_to_not_match_regex\.py', # 2 @@ -136,16 +142,10 @@ exclude = [ 'render/renderer/checkpoint_new_notebook_renderer\.py', # 9 'render/renderer/content_block/content_block\.py', # 5 'render/renderer/content_block/exception_list_content_block\.py', # 4 - 'render/renderer/datasource_new_notebook_renderer\.py', # 4 - 'render/renderer/notebook_renderer\.py', # 2 'render/renderer/page_renderer\.py', # 10 'render/renderer/profiling_results_overview_section_renderer\.py', # 2 'render/renderer/site_builder\.py', # 3 'render/renderer/slack_renderer\.py', # 9 - 'render/renderer/suite_edit_notebook_renderer\.py', # 7 - 'render/renderer/suite_scaffold_notebook_renderer\.py', # 7 - 'render/renderer/v3/suite_edit_notebook_renderer\.py', # 11 - 'render/renderer/v3/suite_profile_notebook_renderer\.py', # 4 'rule_based_profiler/domain_builder/map_metric_column_domain_builder\.py', # 8 'rule_based_profiler/estimators/bootstrap_numeric_range_estimator\.py', # 8 'rule_based_profiler/estimators/kde_numeric_range_estimator\.py', # 7 From f3252380d50b17cbabad1d6cccb0d09e5ff2bbb7 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Tue, 31 Oct 2023 16:04:33 -0400 Subject: [PATCH 04/10] ignore 2 notebook_renderer warnings --- great_expectations/render/renderer/notebook_renderer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/great_expectations/render/renderer/notebook_renderer.py b/great_expectations/render/renderer/notebook_renderer.py index fa636df53f5c..0ce614caafe0 100644 --- a/great_expectations/render/renderer/notebook_renderer.py +++ b/great_expectations/render/renderer/notebook_renderer.py @@ -4,6 +4,7 @@ import nbformat +from great_expectations.compatibility.typing_extensions import override from great_expectations.render.renderer.renderer import Renderer from great_expectations.util import ( convert_json_string_to_be_python_compliant, @@ -48,7 +49,7 @@ def add_code_cell( code = lint_code(code).rstrip("\n") cell = nbformat.v4.new_code_cell(code) - self._notebook["cells"].append(cell) + self._notebook["cells"].append(cell) # type: ignore[index] # _notebook could be None def add_markdown_cell(self, markdown: str) -> None: """ @@ -60,7 +61,7 @@ def add_markdown_cell(self, markdown: str) -> None: Nothing, adds a cell to the class instance notebook """ cell = nbformat.v4.new_markdown_cell(markdown) - self._notebook["cells"].append(cell) + self._notebook["cells"].append(cell) # type: ignore[index] # _notebook could be None @classmethod def write_notebook_to_disk( @@ -75,6 +76,7 @@ def write_notebook_to_disk( with open(notebook_file_path, "w") as f: nbformat.write(notebook, f) + @override def render(self, **kwargs: dict) -> nbformat.NotebookNode: """ Render a notebook from parameters. From 4b04981e0d3bea3b65d17e9a8e93c528c8fa4a8c Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Tue, 31 Oct 2023 16:06:56 -0400 Subject: [PATCH 05/10] import `Template` as `jTemplate` --- great_expectations/render/view/view.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/great_expectations/render/view/view.py b/great_expectations/render/view/view.py index 0fed08e9f55c..97bd62e3e2a2 100644 --- a/great_expectations/render/view/view.py +++ b/great_expectations/render/view/view.py @@ -32,7 +32,8 @@ ) if TYPE_CHECKING: - from jinja2 import BaseLoader, Template + from jinja2 import BaseLoader + from jinja2 import Template as jTemplate class NoOpTemplate: @@ -117,7 +118,7 @@ def render(self, document, template=None, **kwargs): document = document.to_json_dict() return t.render(document, **kwargs) - def _get_template(self, template_str: str) -> Template: + def _get_template(self, template_str: str) -> jTemplate: template = self.env.get_template(template_str) template.globals["now"] = lambda: datetime.datetime.now(datetime.timezone.utc) From eddad7d5a711102e5374ba9a81b66cd8e47f8985 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Tue, 31 Oct 2023 16:10:44 -0400 Subject: [PATCH 06/10] remove `NoOpTemplate` --- great_expectations/render/view/view.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/great_expectations/render/view/view.py b/great_expectations/render/view/view.py index 97bd62e3e2a2..208673a2c7b4 100644 --- a/great_expectations/render/view/view.py +++ b/great_expectations/render/view/view.py @@ -36,11 +36,6 @@ from jinja2 import Template as jTemplate -class NoOpTemplate: - def render(self, document): - return document - - class PrettyPrintTemplate: def render(self, document, indent=2) -> None: print(json.dumps(document, indent=indent)) From 420a4d6bf16d273f3844fb36bf67788fd1e4bb8d Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Wed, 1 Nov 2023 09:56:05 -0400 Subject: [PATCH 07/10] missed one --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cc4272c5c243..39eef9975a51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ exclude = [ 'render/renderer/suite_edit_notebook_renderer\.py', # 7 - This is legacy code and will not be typed. 'render/renderer/suite_scaffold_notebook_renderer\.py', # 7 - This is legacy code and will not be typed. 'render/renderer/datasource_new_notebook_renderer\.py', # 4 - This is legacy code and will not be typed. + 'render/renderer/checkpoint_new_notebook_renderer\.py', # 9 - This is legacy code and will not be typed. # END ALWAYS EXCLUDE SECTION ###################################################### # # ################################################################################# @@ -139,7 +140,6 @@ exclude = [ 'expectations/regex_based_column_map_expectation\.py', # 3 'expectations/row_conditions\.py', # 4 'expectations/set_based_column_map_expectation\.py', # 3 - 'render/renderer/checkpoint_new_notebook_renderer\.py', # 9 'render/renderer/content_block/content_block\.py', # 5 'render/renderer/content_block/exception_list_content_block\.py', # 4 'render/renderer/page_renderer\.py', # 10 From 10f0fde9c11483f7d2894b7c7f8a0993027258ac Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Wed, 1 Nov 2023 10:10:13 -0400 Subject: [PATCH 08/10] missed typing --- great_expectations/render/view/view.py | 63 +++++++++++++++----------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/great_expectations/render/view/view.py b/great_expectations/render/view/view.py index 208673a2c7b4..f773534141f3 100644 --- a/great_expectations/render/view/view.py +++ b/great_expectations/render/view/view.py @@ -5,7 +5,7 @@ import re from collections import OrderedDict from string import Template as pTemplate -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Iterable, Mapping from uuid import uuid4 import mistune @@ -119,8 +119,10 @@ def _get_template(self, template_str: str) -> jTemplate: return template - @contextfilter - def add_data_context_id_to_url(self, jinja_context, url, add_datetime=True): + @contextfilter # type: ignore[misc] # untyped 3rd party decorator + def add_data_context_id_to_url( + self, jinja_context: Any, url: str, add_datetime: bool = True + ) -> str: data_context_id = jinja_context.get("data_context_id") if add_datetime: datetime_iso_string = datetime.datetime.now(datetime.timezone.utc).strftime( @@ -132,15 +134,15 @@ def add_data_context_id_to_url(self, jinja_context, url, add_datetime=True): url += data_context_id return url - @contextfilter + @contextfilter # type: ignore[misc] # untyped 3rd party decorator def render_content_block( # noqa: PLR0911, PLR0913, PLR0912 self, - jinja_context, - content_block, - index=None, - content_block_id=None, + jinja_context: Any, + content_block: str | list | dict | RenderedComponentContent, + index: Any = None, + content_block_id: Any = None, render_to_markdown: bool = False, - ): + ) -> RenderedComponentContent | dict | str: """ :param jinja_context: @@ -202,7 +204,7 @@ def render_content_block( # noqa: PLR0911, PLR0913, PLR0912 ) def render_dict_values( - self, context, dict_, index=None, content_block_id=None + self, context: Any, dict_: dict, index: Any = None, content_block_id: Any = None ) -> None: for key, val in dict_.items(): if key.startswith("_"): @@ -211,18 +213,22 @@ def render_dict_values( context, val, index, content_block_id ) - @contextfilter + @contextfilter # type: ignore[misc] # untyped 3rd party decorator def render_bootstrap_table_data( - self, context, table_data, index=None, content_block_id=None + self, + context: Any, + table_data: Iterable[dict], + index: Any | None = None, + content_block_id: str | None = None, ): for table_data_dict in table_data: self.render_dict_values(context, table_data_dict, index, content_block_id) return table_data - def get_html_escaped_json_string_from_dict(self, source_dict): + def get_html_escaped_json_string_from_dict(self, source_dict: dict) -> str: return json.dumps(source_dict).replace('"', '\\"').replace('"', """) - def attributes_dict_to_html_string(self, attributes_dict, prefix=""): + def attributes_dict_to_html_string(self, attributes_dict: dict, prefix=""): attributes_string = "" if prefix: prefix += "-" @@ -230,7 +236,7 @@ def attributes_dict_to_html_string(self, attributes_dict, prefix=""): attributes_string += f'{prefix}{attribute}="{value}" ' return attributes_string - def render_styling(self, styling): + def render_styling(self, styling: Mapping) -> str: """Adds styling information suitable for an html tag. Example styling block:: @@ -291,7 +297,7 @@ def render_styling(self, styling): return styling_string - def render_styling_from_string_template(self, template): + def render_styling_from_string_template(self, template: dict | OrderedDict) -> str: # NOTE: We should add some kind of type-checking to template """This method is a thin wrapper use to call `render_styling` from within jinja templates.""" if not isinstance(template, (dict, OrderedDict)): @@ -437,7 +443,7 @@ class DefaultJinjaPageView(DefaultJinjaView): _template = "page.j2" @override - def _validate_document(self, document) -> None: + def _validate_document(self, document: RenderedDocumentContent) -> None: assert isinstance(document, RenderedDocumentContent) @@ -449,7 +455,7 @@ class DefaultJinjaSectionView(DefaultJinjaView): _template = "section.j2" @override - def _validate_document(self, document) -> None: + def _validate_document(self, document: Mapping) -> None: assert isinstance( document["section"], dict ) # For now low-level views take dicts @@ -459,7 +465,7 @@ class DefaultJinjaComponentView(DefaultJinjaView): _template = "component.j2" @override - def _validate_document(self, document) -> None: + def _validate_document(self, document: Mapping) -> None: assert isinstance( document["content_block"], dict ) # For now low-level views take dicts @@ -479,7 +485,8 @@ def _validate_document(self, document: RenderedDocumentContent) -> None: _template = "markdown_validation_results_page.j2" - def render(self, document, template=None, **kwargs): + @override + def render(self, document: list, template=None, **kwargs) -> list[str] | str: """ Handle list as well as single document """ @@ -496,7 +503,9 @@ def render(self, document, template=None, **kwargs): return super().render(document=document, template=template, **kwargs) @override - def render_string_template(self, template: pTemplate) -> pTemplate | str: + def render_string_template( + self, template: pTemplate | dict | OrderedDict + ) -> pTemplate | str: """ Render string for markdown rendering. Bold all parameters and perform substitution. Args: @@ -558,15 +567,15 @@ def render_string_template(self, template: pTemplate) -> pTemplate | str: ) @override - @contextfilter + @contextfilter # type: ignore[misc] # untyped 3rd party decorator def render_content_block( # noqa: PLR0913 self, - jinja_context, - content_block, - index=None, - content_block_id=None, + jinja_context: Any, + content_block: Any, + index: Any | None = None, + content_block_id: str | None = None, render_to_markdown: bool = True, - ): + ) -> str: """ Render a content block to markdown using jinja templates. Args: From 2836c7efb82ad33c882ec6010bc020e7506bd5af Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Wed, 1 Nov 2023 10:11:08 -0400 Subject: [PATCH 09/10] missed one --- great_expectations/render/view/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/great_expectations/render/view/view.py b/great_expectations/render/view/view.py index f773534141f3..58fd022082ec 100644 --- a/great_expectations/render/view/view.py +++ b/great_expectations/render/view/view.py @@ -575,7 +575,7 @@ def render_content_block( # noqa: PLR0913 index: Any | None = None, content_block_id: str | None = None, render_to_markdown: bool = True, - ) -> str: + ) -> RenderedComponentContent | dict | str: """ Render a content block to markdown using jinja templates. Args: From c9d098b190877436b2cb9fe6c92c4949adb45179 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Wed, 1 Nov 2023 10:14:51 -0400 Subject: [PATCH 10/10] whitespace --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 39eef9975a51..f1c5f6f59412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,6 @@ exclude = [ # ################################################################################# # TODO: complete typing for the following modules and remove from exclude list # number is the current number of typing errors for the excluded pattern - 'expectations/core/expect_column_values_to_be_of_type\.py', # 12 'expectations/core/expect_column_values_to_not_match_regex_list\.py', # 2 'expectations/core/expect_column_values_to_not_match_regex\.py', # 2