diff --git a/README.md b/README.md index d2af3ccc28..80225cff5c 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,6 @@ Snowflake CLI is an open-source command-line tool explicitly designed for develo With Snowflake CLI, developers can create, manage, update, and view apps running on Snowflake across workloads such as Streamlit in Snowflake, the Snowflake Native App Framework, Snowpark Container Services, and Snowpark. It supports a range of Snowflake features, including user-defined functions, stored procedures, Streamlit in Snowflake, and SQL execution. -**Note**: Snowflake CLI is in Public Preview (PuPr). - Docs: . Quick start: diff --git a/pyproject.toml b/pyproject.toml index 68c30b4302..8df59c740e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "setuptools==75.6.0", 'snowflake.core==1.0.2; python_version < "3.12"', "snowflake-connector-python[secure-local-storage]==3.12.3", - 'snowflake-snowpark-python>=1.15.0;python_version < "3.12"', + 'snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12"', "tomlkit==0.13.2", "typer==0.12.5", "urllib3>=1.24.3,<2.3", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 1c2f44c1a3..e5870050d1 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -8,7 +8,7 @@ requirements-parser==0.11.0 setuptools==75.6.0 snowflake.core==1.0.2; python_version < "3.12" snowflake-connector-python[secure-local-storage]==3.12.3 -snowflake-snowpark-python>=1.15.0;python_version < "3.12" +snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12" tomlkit==0.13.2 typer==0.12.5 urllib3>=1.24.3,<2.3 diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py index 07302e6356..7681f1fab3 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py @@ -33,7 +33,6 @@ from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import ( TemplatesProcessor, ) -from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.metrics import CLICounterField @@ -41,15 +40,7 @@ ProcessorMapping, ) -SNOWPARK_PROCESSOR = "snowpark" -NA_SETUP_PROCESSOR = "native app setup" -TEMPLATES_PROCESSOR = "templates" - -_REGISTERED_PROCESSORS_BY_NAME = { - SNOWPARK_PROCESSOR: SnowparkAnnotationProcessor, - NA_SETUP_PROCESSOR: NativeAppSetupProcessor, - TEMPLATES_PROCESSOR: TemplatesProcessor, -} +ProcessorClassType = type[ArtifactProcessor] class NativeAppCompiler: @@ -66,10 +57,28 @@ def __init__( bundle_ctx: BundleContext, ): self._assert_absolute_paths(bundle_ctx) + self._processor_classes_by_name: Dict[str, ProcessorClassType] = {} self._bundle_ctx = bundle_ctx # dictionary of all processors created and shared between different artifact objects. self.cached_processors: Dict[str, ArtifactProcessor] = {} + self.register(SnowparkAnnotationProcessor) + self.register(NativeAppSetupProcessor) + self.register(TemplatesProcessor) + + def register(self, processor_cls: ProcessorClassType): + """ + Registers a processor class to enable. + """ + + name = getattr(processor_cls, "NAME", None) + assert name is not None + + if name in self._processor_classes_by_name: + raise ValueError(f"Processor {name} is already registered") + + self._processor_classes_by_name[str(name)] = processor_cls + @staticmethod def _assert_absolute_paths(bundle_ctx: BundleContext): for name in ["Project", "Deploy", "Bundle", "Generated"]: @@ -128,8 +137,8 @@ def _try_create_processor( if current_processor is not None: return current_processor - processor_factory = _REGISTERED_PROCESSORS_BY_NAME.get(processor_name) - if processor_factory is None: + processor_cls = self._processor_classes_by_name.get(processor_name) + if processor_cls is None: # No registered processor with the specified name return None @@ -141,7 +150,7 @@ def _try_create_processor( processor_ctx.generated_root = ( self._bundle_ctx.generated_root / processor_subdirectory ) - current_processor = processor_factory(processor_ctx) + current_processor = processor_cls(processor_ctx) self.cached_processors[processor_name] = current_processor return current_processor @@ -154,6 +163,18 @@ def _should_invoke_processors(self): return False def _is_enabled(self, processor: ProcessorMapping) -> bool: - if processor.name.lower() == NA_SETUP_PROCESSOR: - return FeatureFlag.ENABLE_NATIVE_APP_PYTHON_SETUP.is_enabled() - return True + """ + Determines is a process is enabled. All processors are considered enabled + unless they are explicitly disabled, typically via a feature flag. + """ + processor_name = processor.name.lower() + processor_cls = self._processor_classes_by_name.get(processor_name) + if processor_cls is None: + # Unknown processor, consider it enabled, even though trying to + # invoke it later will raise an exception + return True + + # if the processor class defines a static method named "is_enabled", then + # call it. Otherwise, it's considered enabled by default. + is_enabled_fn = getattr(processor_cls, "is_enabled", lambda: True) + return is_enabled_fn() diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py index 06be6f01e8..b643f66304 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py @@ -36,6 +36,7 @@ SandboxEnvBuilder, execute_script_in_sandbox, ) +from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag from snowflake.cli._plugins.stage.diff import to_stage_path from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( @@ -74,9 +75,15 @@ def safe_set(d: dict, *keys: str, **kwargs) -> None: class NativeAppSetupProcessor(ArtifactProcessor): + NAME = "native app setup" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + @staticmethod + def is_enabled() -> bool: + return FeatureFlag.ENABLE_NATIVE_APP_PYTHON_SETUP.is_enabled() + def process( self, artifact_to_process: PathMapping, diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py index 0241899388..58a9eb2baa 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py @@ -164,6 +164,8 @@ class SnowparkAnnotationProcessor(ArtifactProcessor): and generate SQL code for creation of extension functions based on those discovered objects. """ + NAME = "snowpark" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index 9e38eecc2c..b6984f67c2 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -49,6 +49,8 @@ class TemplatesProcessor(ArtifactProcessor): Processor class to perform template expansion on all relevant artifacts (specified in the project definition file). """ + NAME = "templates" + def expand_templates_in_file( self, src: Path, dest: Path, template_context: dict[str, Any] | None = None ) -> None: @@ -58,38 +60,45 @@ def expand_templates_in_file( if src.is_dir(): return - with self.edit_file(dest) as file: - if not has_client_side_templates(file.contents) and not ( - _is_sql_file(dest) and has_sql_templates(file.contents) - ): - return - - src_file_name = src.relative_to(self._bundle_ctx.project_root) - cc.step(f"Expanding templates in {src_file_name}") - with cc.indented(): - try: - jinja_env = ( - choose_sql_jinja_env_based_on_template_syntax( - file.contents, reference_name=src_file_name + src_file_name = src.relative_to(self._bundle_ctx.project_root) + + try: + with self.edit_file(dest) as file: + if not has_client_side_templates(file.contents) and not ( + _is_sql_file(dest) and has_sql_templates(file.contents) + ): + return + cc.step(f"Expanding templates in {src_file_name}") + with cc.indented(): + try: + jinja_env = ( + choose_sql_jinja_env_based_on_template_syntax( + file.contents, reference_name=src_file_name + ) + if _is_sql_file(dest) + else get_client_side_jinja_env() + ) + expanded_template = jinja_env.from_string(file.contents).render( + template_context or get_cli_context().template_context ) - if _is_sql_file(dest) - else get_client_side_jinja_env() - ) - expanded_template = jinja_env.from_string(file.contents).render( - template_context or get_cli_context().template_context - ) - - # For now, we are printing the source file path in the error message - # instead of the destination file path to make it easier for the user - # to identify the file that has the error, and edit the correct file. - except jinja2.TemplateSyntaxError as e: - raise InvalidTemplateInFileError(src_file_name, e, e.lineno) from e - - except jinja2.UndefinedError as e: - raise InvalidTemplateInFileError(src_file_name, e) from e - - if expanded_template != file.contents: - file.edited_contents = expanded_template + + # For now, we are printing the source file path in the error message + # instead of the destination file path to make it easier for the user + # to identify the file that has the error, and edit the correct file. + except jinja2.TemplateSyntaxError as e: + raise InvalidTemplateInFileError( + src_file_name, e, e.lineno + ) from e + + except jinja2.UndefinedError as e: + raise InvalidTemplateInFileError(src_file_name, e) from e + + if expanded_template != file.contents: + file.edited_contents = expanded_template + except UnicodeDecodeError as err: + cc.warning( + f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." + ) @span("templates_processor") def process( diff --git a/src/snowflake/cli/api/project/definition_conversion.py b/src/snowflake/cli/api/project/definition_conversion.py index 1f76e998d2..7834586a07 100644 --- a/src/snowflake/cli/api/project/definition_conversion.py +++ b/src/snowflake/cli/api/project/definition_conversion.py @@ -13,7 +13,6 @@ bundle_artifacts, ) from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext -from snowflake.cli._plugins.nativeapp.codegen.compiler import TEMPLATES_PROCESSOR from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import ( TemplatesProcessor, ) @@ -457,7 +456,7 @@ def _convert_templates_in_files( artifact for artifact in pkg_model.artifacts for processor in artifact.processors - if processor.name == TEMPLATES_PROCESSOR + if processor.name.lower() == TemplatesProcessor.NAME ] if not in_memory and artifacts_to_template: metrics.set_counter(CLICounterField.TEMPLATES_PROCESSOR, 1) diff --git a/tests/nativeapp/codegen/templating/test_templates_processor.py b/tests/nativeapp/codegen/templating/test_templates_processor.py index 21c7160d06..65eb5b3dac 100644 --- a/tests/nativeapp/codegen/templating/test_templates_processor.py +++ b/tests/nativeapp/codegen/templating/test_templates_processor.py @@ -28,7 +28,10 @@ from snowflake.cli.api.exceptions import InvalidTemplate from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping -from tests.nativeapp.utils import CLI_GLOBAL_TEMPLATE_CONTEXT +from tests.nativeapp.utils import ( + CLI_GLOBAL_TEMPLATE_CONTEXT, + TEMPLATE_PROCESSOR, +) @dataclass @@ -213,3 +216,24 @@ def test_file_with_undefined_variable(): assert "does not contain a valid template" in str(e.value) assert bundle_result.output_files[0].is_symlink() assert bundle_result.output_files[0].read_text() == file_contents[0] + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {}) +@mock.patch(f"{TEMPLATE_PROCESSOR}.cc.warning") +def test_expand_templates_in_file_unicode_decode_error(mock_cc_warning): + file_name = ["test_file.txt"] + file_contents = ["This is a test file"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_name, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + with mock.patch( + f"{TEMPLATE_PROCESSOR}.TemplatesProcessor.edit_file", + side_effect=UnicodeDecodeError("utf-8", b"", 0, 1, "invalid start byte"), + ): + src_path = Path( + bundle_result.bundle_ctx.project_root / "src" / file_name[0] + ).relative_to(bundle_result.bundle_ctx.project_root) + templates_processor.process(bundle_result.artifact_to_process, None) + mock_cc_warning.assert_called_once_with( + f"Could not read file {src_path}, error: invalid start byte. Skipping this file." + ) diff --git a/tests/nativeapp/codegen/test_compiler.py b/tests/nativeapp/codegen/test_compiler.py index 4ec64ef983..7da3382588 100644 --- a/tests/nativeapp/codegen/test_compiler.py +++ b/tests/nativeapp/codegen/test_compiler.py @@ -13,10 +13,12 @@ # limitations under the License. import re from pathlib import Path +from typing import Optional import pytest from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import ( + ArtifactProcessor, UnsupportedArtifactProcessorError, ) from snowflake.cli._plugins.nativeapp.codegen.compiler import NativeAppCompiler @@ -29,6 +31,10 @@ from snowflake.cli.api.project.schemas.project_definition import ( build_project_definition, ) +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( + PathMapping, + ProcessorMapping, +) @pytest.fixture() @@ -114,3 +120,35 @@ def test_find_and_execute_processors_exception(test_proj_def, test_compiler): with pytest.raises(UnsupportedArtifactProcessorError): test_compiler.compile_artifacts() + + +class TestProcessor(ArtifactProcessor): + NAME = "test_processor" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert False # never invoked + + @staticmethod + def is_enabled(): + return False + + def process( + self, + artifact_to_process: PathMapping, + processor_mapping: Optional[ProcessorMapping], + **kwargs, + ) -> None: + assert False # never invoked + + +def test_skips_disabled_processors(test_proj_def, test_compiler): + pkg_model = test_proj_def.entities["pkg"] + pkg_model.artifacts = [ + {"dest": "./", "src": "app/*", "processors": ["test_processor"]} + ] + test_compiler = NativeAppCompiler(_get_bundle_context(pkg_model)) + test_compiler.register(TestProcessor) + + # TestProcessor is never invoked, otherwise calling its methods will make the test fail + test_compiler.compile_artifacts() diff --git a/tests/nativeapp/utils.py b/tests/nativeapp/utils.py index 4a614bbbda..2eb4c852d1 100644 --- a/tests/nativeapp/utils.py +++ b/tests/nativeapp/utils.py @@ -62,6 +62,10 @@ f"{APP_PACKAGE_ENTITY}.verify_project_distribution" ) +CODE_GEN = "snowflake.cli._plugins.nativeapp.codegen" +TEMPLATE_PROCESSOR = f"{CODE_GEN}.templates.templates_processor" +ARTIFACT_PROCESSOR = f"{CODE_GEN}.artifact_processor" + SQL_EXECUTOR_EXECUTE = f"{API_MODULE}.sql_execution.BaseSqlExecutor.execute_query" SQL_EXECUTOR_EXECUTE_QUERIES = ( f"{API_MODULE}.sql_execution.BaseSqlExecutor.execute_queries" diff --git a/tests/test_connection.py b/tests/test_connection.py index 8345c1c3cd..ecb81f6da2 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1219,16 +1219,22 @@ def test_generate_jwt_with_passphrase( result.output == "Enter private key file password (press enter for empty) []: \nfunny token\n" ) - mocked_get_token.has_calls( - mock.call( - user="FooBar", account="account1", privatekey_path=str(f), key_password=None - ), - mock.call( - user="FooBar", - account="account1", - privatekey_path=str(f), - key_password=passphrase, - ), + mocked_get_token.assert_has_calls( + [ + mock.call( + user="FooBar", + account="account1", + privatekey_path=str(f), + key_password=None, + ), + mock.call( + user="FooBar", + account="account1", + privatekey_path=str(f), + key_password=passphrase, + ), + ], + any_order=True, ) diff --git a/tests_e2e/conftest.py b/tests_e2e/conftest.py index 7bca4d9d1c..38bbb8327c 100644 --- a/tests_e2e/conftest.py +++ b/tests_e2e/conftest.py @@ -140,7 +140,7 @@ def _install_snowcli_with_external_plugin( ) # Required by snowpark example tests - _pip_install(python, "snowflake-snowpark-python") + _pip_install(python, "snowflake-snowpark-python[pandas]==1.25.0") def _python_path(venv_path: Path) -> Path: diff --git a/tests_integration/helpers/test_v1_to_v2.py b/tests_integration/helpers/test_v1_to_v2.py index bfc594ff0b..a4aae93244 100644 --- a/tests_integration/helpers/test_v1_to_v2.py +++ b/tests_integration/helpers/test_v1_to_v2.py @@ -2,7 +2,9 @@ import pytest -from snowflake.cli._plugins.nativeapp.codegen.compiler import TEMPLATES_PROCESSOR +from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import ( + TemplatesProcessor, +) from tests.nativeapp.factories import ProjectV11Factory @@ -44,10 +46,10 @@ def test_v1_to_v2_converts_templates_in_files(temp_dir, runner): pdf__native_app__package__name="my_pkg", pdf__native_app__application__name="my_app", pdf__native_app__artifacts=[ - dict(src="templated.txt", processors=[TEMPLATES_PROCESSOR]), + dict(src="templated.txt", processors=[TemplatesProcessor.NAME]), dict(src="untemplated.txt"), - dict(src="app/*", processors=[TEMPLATES_PROCESSOR]), - dict(src="nested/*", processors=[TEMPLATES_PROCESSOR]), + dict(src="app/*", processors=[TemplatesProcessor.NAME]), + dict(src="nested/*", processors=[TemplatesProcessor.NAME]), ], files={ filename: source_contents