Merge branch 'main' into gbloom-package-children-feature-flag

README.md

@@ -26,8 +26,6 @@ Snowflake CLI is an open-source command-line tool explicitly designed for develo
 
 With Snowflake CLI, developers can create, manage, update, and view apps running on Snowflake across workloads such as Streamlit in Snowflake, the Snowflake Native App Framework, Snowpark Container Services, and Snowpark. It supports a range of Snowflake features, including user-defined functions, stored procedures, Streamlit in Snowflake, and SQL execution.
 
-**Note**: Snowflake CLI is in Public Preview (PuPr).
-
 Docs: <https://docs.snowflake.com/en/developer-guide/snowflake-cli-v2/index>.
 
 Quick start: <https://quickstarts.snowflake.com/guide/getting-started-with-snowflake-cli>

pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
   "setuptools==75.6.0",
   'snowflake.core==1.0.2; python_version < "3.12"',
   "snowflake-connector-python[secure-local-storage]==3.12.3",
-  'snowflake-snowpark-python>=1.15.0;python_version < "3.12"',
+  'snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12"',
   "tomlkit==0.13.2",
   "typer==0.12.5",
   "urllib3>=1.24.3,<2.3",

snyk/requirements.txt

@@ -8,7 +8,7 @@ requirements-parser==0.11.0
 setuptools==75.6.0
 snowflake.core==1.0.2; python_version < "3.12"
 snowflake-connector-python[secure-local-storage]==3.12.3
-snowflake-snowpark-python>=1.15.0;python_version < "3.12"
+snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12"
 tomlkit==0.13.2
 typer==0.12.5
 urllib3>=1.24.3,<2.3

src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py

@@ -33,23 +33,14 @@
 from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import (
     TemplatesProcessor,
 )
-from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag
 from snowflake.cli.api.cli_global_context import get_cli_context
 from snowflake.cli.api.console import cli_console as cc
 from snowflake.cli.api.metrics import CLICounterField
 from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import (
     ProcessorMapping,
 )
 
-SNOWPARK_PROCESSOR = "snowpark"
-NA_SETUP_PROCESSOR = "native app setup"
-TEMPLATES_PROCESSOR = "templates"
-
-_REGISTERED_PROCESSORS_BY_NAME = {
-    SNOWPARK_PROCESSOR: SnowparkAnnotationProcessor,
-    NA_SETUP_PROCESSOR: NativeAppSetupProcessor,
-    TEMPLATES_PROCESSOR: TemplatesProcessor,
-}
+ProcessorClassType = type[ArtifactProcessor]
 
 
 class NativeAppCompiler:
@@ -66,10 +57,28 @@ def __init__(
         bundle_ctx: BundleContext,
     ):
         self._assert_absolute_paths(bundle_ctx)
+        self._processor_classes_by_name: Dict[str, ProcessorClassType] = {}
         self._bundle_ctx = bundle_ctx
         # dictionary of all processors created and shared between different artifact objects.
         self.cached_processors: Dict[str, ArtifactProcessor] = {}
 
+        self.register(SnowparkAnnotationProcessor)
+        self.register(NativeAppSetupProcessor)
+        self.register(TemplatesProcessor)
+
+    def register(self, processor_cls: ProcessorClassType):
+        """
+        Registers a processor class to enable.
+        """
+
+        name = getattr(processor_cls, "NAME", None)
+        assert name is not None
+
+        if name in self._processor_classes_by_name:
+            raise ValueError(f"Processor {name} is already registered")
+
+        self._processor_classes_by_name[str(name)] = processor_cls
+
     @staticmethod
     def _assert_absolute_paths(bundle_ctx: BundleContext):
         for name in ["Project", "Deploy", "Bundle", "Generated"]:
@@ -128,8 +137,8 @@ def _try_create_processor(
         if current_processor is not None:
             return current_processor
 
-        processor_factory = _REGISTERED_PROCESSORS_BY_NAME.get(processor_name)
-        if processor_factory is None:
+        processor_cls = self._processor_classes_by_name.get(processor_name)
+        if processor_cls is None:
             # No registered processor with the specified name
             return None
 
@@ -141,7 +150,7 @@ def _try_create_processor(
         processor_ctx.generated_root = (
             self._bundle_ctx.generated_root / processor_subdirectory
         )
-        current_processor = processor_factory(processor_ctx)
+        current_processor = processor_cls(processor_ctx)
         self.cached_processors[processor_name] = current_processor
 
         return current_processor
@@ -154,6 +163,18 @@ def _should_invoke_processors(self):
         return False
 
     def _is_enabled(self, processor: ProcessorMapping) -> bool:
-        if processor.name.lower() == NA_SETUP_PROCESSOR:
-            return FeatureFlag.ENABLE_NATIVE_APP_PYTHON_SETUP.is_enabled()
-        return True
+        """
+        Determines is a process is enabled. All processors are considered enabled
+        unless they are explicitly disabled, typically via a feature flag.
+        """
+        processor_name = processor.name.lower()
+        processor_cls = self._processor_classes_by_name.get(processor_name)
+        if processor_cls is None:
+            # Unknown processor, consider it enabled, even though trying to
+            # invoke it later will raise an exception
+            return True
+
+        # if the processor class defines a static method named "is_enabled", then
+        # call it. Otherwise, it's considered enabled by default.
+        is_enabled_fn = getattr(processor_cls, "is_enabled", lambda: True)
+        return is_enabled_fn()

src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py

@@ -36,6 +36,7 @@
     SandboxEnvBuilder,
     execute_script_in_sandbox,
 )
+from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag
 from snowflake.cli._plugins.stage.diff import to_stage_path
 from snowflake.cli.api.console import cli_console as cc
 from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import (
@@ -74,9 +75,15 @@ def safe_set(d: dict, *keys: str, **kwargs) -> None:
 
 
 class NativeAppSetupProcessor(ArtifactProcessor):
+    NAME = "native app setup"
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    @staticmethod
+    def is_enabled() -> bool:
+        return FeatureFlag.ENABLE_NATIVE_APP_PYTHON_SETUP.is_enabled()
+
     def process(
         self,
         artifact_to_process: PathMapping,

src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py

@@ -164,6 +164,8 @@ class SnowparkAnnotationProcessor(ArtifactProcessor):
     and generate SQL code for creation of extension functions based on those discovered objects.
     """
 
+    NAME = "snowpark"
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 

src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py

@@ -49,6 +49,8 @@ class TemplatesProcessor(ArtifactProcessor):
     Processor class to perform template expansion on all relevant artifacts (specified in the project definition file).
     """
 
+    NAME = "templates"
+
     def expand_templates_in_file(
         self, src: Path, dest: Path, template_context: dict[str, Any] | None = None
     ) -> None:
@@ -58,38 +60,45 @@ def expand_templates_in_file(
         if src.is_dir():
             return
 
-        with self.edit_file(dest) as file:
-            if not has_client_side_templates(file.contents) and not (
-                _is_sql_file(dest) and has_sql_templates(file.contents)
-            ):
-                return
-
-            src_file_name = src.relative_to(self._bundle_ctx.project_root)
-            cc.step(f"Expanding templates in {src_file_name}")
-            with cc.indented():
-                try:
-                    jinja_env = (
-                        choose_sql_jinja_env_based_on_template_syntax(
-                            file.contents, reference_name=src_file_name
+        src_file_name = src.relative_to(self._bundle_ctx.project_root)
+
+        try:
+            with self.edit_file(dest) as file:
+                if not has_client_side_templates(file.contents) and not (
+                    _is_sql_file(dest) and has_sql_templates(file.contents)
+                ):
+                    return
+                cc.step(f"Expanding templates in {src_file_name}")
+                with cc.indented():
+                    try:
+                        jinja_env = (
+                            choose_sql_jinja_env_based_on_template_syntax(
+                                file.contents, reference_name=src_file_name
+                            )
+                            if _is_sql_file(dest)
+                            else get_client_side_jinja_env()
+                        )
+                        expanded_template = jinja_env.from_string(file.contents).render(
+                            template_context or get_cli_context().template_context
                         )
-                        if _is_sql_file(dest)
-                        else get_client_side_jinja_env()
-                    )
-                    expanded_template = jinja_env.from_string(file.contents).render(
-                        template_context or get_cli_context().template_context
-                    )
-
-                # For now, we are printing the source file path in the error message
-                # instead of the destination file path to make it easier for the user
-                # to identify the file that has the error, and edit the correct file.
-                except jinja2.TemplateSyntaxError as e:
-                    raise InvalidTemplateInFileError(src_file_name, e, e.lineno) from e
-
-                except jinja2.UndefinedError as e:
-                    raise InvalidTemplateInFileError(src_file_name, e) from e
-
-                if expanded_template != file.contents:
-                    file.edited_contents = expanded_template
+
+                    # For now, we are printing the source file path in the error message
+                    # instead of the destination file path to make it easier for the user
+                    # to identify the file that has the error, and edit the correct file.
+                    except jinja2.TemplateSyntaxError as e:
+                        raise InvalidTemplateInFileError(
+                            src_file_name, e, e.lineno
+                        ) from e
+
+                    except jinja2.UndefinedError as e:
+                        raise InvalidTemplateInFileError(src_file_name, e) from e
+
+                    if expanded_template != file.contents:
+                        file.edited_contents = expanded_template
+        except UnicodeDecodeError as err:
+            cc.warning(
+                f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file."
+            )
 
     @span("templates_processor")
     def process(

src/snowflake/cli/api/project/definition_conversion.py

@@ -13,7 +13,6 @@
     bundle_artifacts,
 )
 from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
-from snowflake.cli._plugins.nativeapp.codegen.compiler import TEMPLATES_PROCESSOR
 from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import (
     TemplatesProcessor,
 )
@@ -457,7 +456,7 @@ def _convert_templates_in_files(
             artifact
             for artifact in pkg_model.artifacts
             for processor in artifact.processors
-            if processor.name == TEMPLATES_PROCESSOR
+            if processor.name.lower() == TemplatesProcessor.NAME
         ]
         if not in_memory and artifacts_to_template:
             metrics.set_counter(CLICounterField.TEMPLATES_PROCESSOR, 1)

tests/nativeapp/codegen/templating/test_templates_processor.py

@@ -28,7 +28,10 @@
 from snowflake.cli.api.exceptions import InvalidTemplate
 from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping
 
-from tests.nativeapp.utils import CLI_GLOBAL_TEMPLATE_CONTEXT
+from tests.nativeapp.utils import (
+    CLI_GLOBAL_TEMPLATE_CONTEXT,
+    TEMPLATE_PROCESSOR,
+)
 
 
 @dataclass
@@ -213,3 +216,24 @@ def test_file_with_undefined_variable():
         assert "does not contain a valid template" in str(e.value)
         assert bundle_result.output_files[0].is_symlink()
         assert bundle_result.output_files[0].read_text() == file_contents[0]
+
+
+@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {})
+@mock.patch(f"{TEMPLATE_PROCESSOR}.cc.warning")
+def test_expand_templates_in_file_unicode_decode_error(mock_cc_warning):
+    file_name = ["test_file.txt"]
+    file_contents = ["This is a test file"]
+    with TemporaryDirectory() as tmp_dir:
+        bundle_result = bundle_files(tmp_dir, file_name, file_contents)
+        templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx)
+        with mock.patch(
+            f"{TEMPLATE_PROCESSOR}.TemplatesProcessor.edit_file",
+            side_effect=UnicodeDecodeError("utf-8", b"", 0, 1, "invalid start byte"),
+        ):
+            src_path = Path(
+                bundle_result.bundle_ctx.project_root / "src" / file_name[0]
+            ).relative_to(bundle_result.bundle_ctx.project_root)
+            templates_processor.process(bundle_result.artifact_to_process, None)
+            mock_cc_warning.assert_called_once_with(
+                f"Could not read file {src_path}, error: invalid start byte. Skipping this file."
+            )

tests/nativeapp/codegen/test_compiler.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 import re
 from pathlib import Path
+from typing import Optional
 
 import pytest
 from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
 from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import (
+    ArtifactProcessor,
     UnsupportedArtifactProcessorError,
 )
 from snowflake.cli._plugins.nativeapp.codegen.compiler import NativeAppCompiler
@@ -29,6 +31,10 @@
 from snowflake.cli.api.project.schemas.project_definition import (
     build_project_definition,
 )
+from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import (
+    PathMapping,
+    ProcessorMapping,
+)
 
 
 @pytest.fixture()
@@ -114,3 +120,35 @@ def test_find_and_execute_processors_exception(test_proj_def, test_compiler):
 
     with pytest.raises(UnsupportedArtifactProcessorError):
         test_compiler.compile_artifacts()
+
+
+class TestProcessor(ArtifactProcessor):
+    NAME = "test_processor"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert False  # never invoked
+
+    @staticmethod
+    def is_enabled():
+        return False
+
+    def process(
+        self,
+        artifact_to_process: PathMapping,
+        processor_mapping: Optional[ProcessorMapping],
+        **kwargs,
+    ) -> None:
+        assert False  # never invoked
+
+
+def test_skips_disabled_processors(test_proj_def, test_compiler):
+    pkg_model = test_proj_def.entities["pkg"]
+    pkg_model.artifacts = [
+        {"dest": "./", "src": "app/*", "processors": ["test_processor"]}
+    ]
+    test_compiler = NativeAppCompiler(_get_bundle_context(pkg_model))
+    test_compiler.register(TestProcessor)
+
+    # TestProcessor is never invoked, otherwise calling its methods will make the test fail
+    test_compiler.compile_artifacts()

tests/nativeapp/utils.py

@@ -62,6 +62,10 @@
     f"{APP_PACKAGE_ENTITY}.verify_project_distribution"
 )
 
+CODE_GEN = "snowflake.cli._plugins.nativeapp.codegen"
+TEMPLATE_PROCESSOR = f"{CODE_GEN}.templates.templates_processor"
+ARTIFACT_PROCESSOR = f"{CODE_GEN}.artifact_processor"
+
 SQL_EXECUTOR_EXECUTE = f"{API_MODULE}.sql_execution.BaseSqlExecutor.execute_query"
 SQL_EXECUTOR_EXECUTE_QUERIES = (
     f"{API_MODULE}.sql_execution.BaseSqlExecutor.execute_queries"

tests/test_connection.py

@@ -1219,16 +1219,22 @@ def test_generate_jwt_with_passphrase(
         result.output
         == "Enter private key file password (press enter for empty) []: \nfunny token\n"
     )
-    mocked_get_token.has_calls(
-        mock.call(
-            user="FooBar", account="account1", privatekey_path=str(f), key_password=None
-        ),
-        mock.call(
-            user="FooBar",
-            account="account1",
-            privatekey_path=str(f),
-            key_password=passphrase,
-        ),
+    mocked_get_token.assert_has_calls(
+        [
+            mock.call(
+                user="FooBar",
+                account="account1",
+                privatekey_path=str(f),
+                key_password=None,
+            ),
+            mock.call(
+                user="FooBar",
+                account="account1",
+                privatekey_path=str(f),
+                key_password=passphrase,
+            ),
+        ],
+        any_order=True,
     )