Decouple project definition object from artifact processors (#1374)

* decouple pdf from processors

* pass bundle context

* extract bundle context to a module

* move get_bundle_context() to NAPM

src/snowflake/cli/plugins/nativeapp/bundle_context.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import (
+    List,
+)
+
+from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping
+
+
+@dataclass
+class BundleContext:
+    package_name: str
+    artifacts: List[PathMapping]
+    project_root: Path
+    bundle_root: Path
+    deploy_root: Path
+    generated_root: Path

src/snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py

@@ -23,7 +23,7 @@
     PathMapping,
     ProcessorMapping,
 )
-from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel
+from snowflake.cli.plugins.nativeapp.bundle_context import BundleContext
 
 
 class UnsupportedArtifactProcessorError(ClickException):
@@ -74,9 +74,9 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 class ArtifactProcessor(ABC):
     def __init__(
         self,
-        na_project: NativeAppProjectModel,
+        bundle_ctx: BundleContext,
     ) -> None:
-        self._na_project = na_project
+        self._bundle_ctx = bundle_ctx
 
     @abstractmethod
     def process(

src/snowflake/cli/plugins/nativeapp/codegen/compiler.py

@@ -20,6 +20,7 @@
 from snowflake.cli.api.project.schemas.native_app.path_mapping import (
     ProcessorMapping,
 )
+from snowflake.cli.plugins.nativeapp.bundle_context import BundleContext
 from snowflake.cli.plugins.nativeapp.codegen.artifact_processor import (
     ArtifactProcessor,
     UnsupportedArtifactProcessorError,
@@ -31,7 +32,6 @@
     SnowparkAnnotationProcessor,
 )
 from snowflake.cli.plugins.nativeapp.feature_flags import FeatureFlag
-from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel
 
 SNOWPARK_PROCESSOR = "snowpark"
 NA_SETUP_PROCESSOR = "native-app-setup"
@@ -53,9 +53,9 @@ class NativeAppCompiler:
 
     def __init__(
         self,
-        na_project: NativeAppProjectModel,
+        bundle_ctx: BundleContext,
     ):
-        self._na_project = na_project
+        self._bundle_ctx = bundle_ctx
         # dictionary of all processors created and shared between different artifact objects.
         self.cached_processors: Dict[str, ArtifactProcessor] = {}
 
@@ -69,12 +69,12 @@ def compile_artifacts(self):
             return
 
         with cc.phase("Invoking artifact processors"):
-            if self._na_project.generated_root.exists():
+            if self._bundle_ctx.generated_root.exists():
                 raise ClickException(
-                    f"Path {self._na_project.generated_root} already exists. Please choose a different name for your generated directory in the project definition file."
+                    f"Path {self._bundle_ctx.generated_root} already exists. Please choose a different name for your generated directory in the project definition file."
                 )
 
-            for artifact in self._na_project.artifacts:
+            for artifact in self._bundle_ctx.artifacts:
                 for processor in artifact.processors:
                     if self._is_enabled(processor):
                         artifact_processor = self._try_create_processor(
@@ -110,15 +110,13 @@ def _try_create_processor(
             # No registered processor with the specified name
             return None
 
-        current_processor = processor_factory(
-            na_project=self._na_project,
-        )
+        current_processor = processor_factory(self._bundle_ctx)
         self.cached_processors[processor_name] = current_processor
 
         return current_processor
 
     def _should_invoke_processors(self):
-        for artifact in self._na_project.artifacts:
+        for artifact in self._bundle_ctx.artifacts:
             for processor in artifact.processors:
                 if self._is_enabled(processor):
                     return True

src/snowflake/cli/plugins/nativeapp/codegen/setup/native_app_setup_processor.py

@@ -35,19 +35,15 @@
     SandboxEnvBuilder,
     execute_script_in_sandbox,
 )
-from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel
 from snowflake.cli.plugins.stage.diff import to_stage_path
 
 DEFAULT_TIMEOUT = 30
 DRIVER_PATH = Path(__file__).parent / "setup_driver.py.source"
 
 
 class NativeAppSetupProcessor(ArtifactProcessor):
-    def __init__(
-        self,
-        na_project: NativeAppProjectModel,
-    ):
-        super().__init__(na_project=na_project)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
 
     def process(
         self,
@@ -59,8 +55,8 @@ def process(
         Processes a Python setup script and generates the corresponding SQL commands.
         """
         bundle_map = BundleMap(
-            project_root=self._na_project.project_root,
-            deploy_root=self._na_project.deploy_root,
+            project_root=self._bundle_ctx.project_root,
+            deploy_root=self._bundle_ctx.deploy_root,
         )
         bundle_map.add(artifact_to_process)
 
@@ -73,7 +69,7 @@ def process(
             absolute=True, expand_directories=True, predicate=is_python_file_artifact
         ):
             cc.message(
-                f"Found Python setup file: {src_file.relative_to(self._na_project.project_root)}"
+                f"Found Python setup file: {src_file.relative_to(self._bundle_ctx.project_root)}"
             )
             files_to_process.append(src_file)
 
@@ -85,17 +81,17 @@ def _execute_in_sandbox(self, py_files: List[Path]) -> dict:
         cc.step(f"Processing {file_count} setup file{'s' if file_count > 1 else ''}")
 
         env_vars = {
-            "_SNOWFLAKE_CLI_PROJECT_PATH": str(self._na_project.project_root),
+            "_SNOWFLAKE_CLI_PROJECT_PATH": str(self._bundle_ctx.project_root),
             "_SNOWFLAKE_CLI_SETUP_FILES": os.pathsep.join(map(str, py_files)),
-            "_SNOWFLAKE_CLI_APP_NAME": str(self._na_project.package_name),
+            "_SNOWFLAKE_CLI_APP_NAME": str(self._bundle_ctx.package_name),
             "_SNOWFLAKE_CLI_SQL_DEST_DIR": str(self.generated_root),
         }
 
         try:
             result = execute_script_in_sandbox(
                 script_source=DRIVER_PATH.read_text(),
                 env_type=ExecutionEnvironmentType.VENV,
-                cwd=self._na_project.bundle_root,
+                cwd=self._bundle_ctx.bundle_root,
                 timeout=DEFAULT_TIMEOUT,
                 path=self.sandbox_root,
                 env_vars=env_vars,
@@ -123,7 +119,7 @@ def _generate_setup_sql(self, sql_file_mappings: dict) -> None:
 
         cc.step("Patching setup script")
         setup_file_path = find_setup_script_file(
-            deploy_root=self._na_project.deploy_root
+            deploy_root=self._bundle_ctx.deploy_root
         )
         with self.edit_file(setup_file_path) as f:
             new_contents = [f.contents]
@@ -132,30 +128,30 @@ def _generate_setup_sql(self, sql_file_mappings: dict) -> None:
                 schemas_file = generated_root / sql_file_mappings["schemas"]
                 new_contents.insert(
                     0,
-                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(schemas_file.relative_to(self._na_project.deploy_root))}';",
+                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(schemas_file.relative_to(self._bundle_ctx.deploy_root))}';",
                 )
 
             if sql_file_mappings["compute_pools"]:
                 compute_pools_file = generated_root / sql_file_mappings["compute_pools"]
                 new_contents.append(
-                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(compute_pools_file.relative_to(self._na_project.deploy_root))}';"
+                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(compute_pools_file.relative_to(self._bundle_ctx.deploy_root))}';"
                 )
 
             if sql_file_mappings["services"]:
                 services_file = generated_root / sql_file_mappings["services"]
                 new_contents.append(
-                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(services_file.relative_to(self._na_project.deploy_root))}';"
+                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(services_file.relative_to(self._bundle_ctx.deploy_root))}';"
                 )
 
             f.edited_contents = "\n".join(new_contents)
 
     @property
     def sandbox_root(self):
-        return self._na_project.bundle_root / "setup_py_venv"
+        return self._bundle_ctx.bundle_root / "setup_py_venv"
 
     @property
     def generated_root(self):
-        return self._na_project.generated_root / "setup_py"
+        return self._bundle_ctx.generated_root / "setup_py"
 
     def _create_or_update_sandbox(self):
         sandbox_root = self.sandbox_root
@@ -164,7 +160,7 @@ def _create_or_update_sandbox(self):
             cc.step("Virtual environment found")
         else:
             cc.step(
-                f"Creating virtual environment in {sandbox_root.relative_to(self._na_project.project_root)}"
+                f"Creating virtual environment in {sandbox_root.relative_to(self._bundle_ctx.project_root)}"
             )
         env_builder.ensure_created()
 

src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py

@@ -53,7 +53,6 @@
     ExtensionFunctionTypeEnum,
     NativeAppExtensionFunction,
 )
-from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel
 from snowflake.cli.plugins.stage.diff import to_stage_path
 
 DEFAULT_TIMEOUT = 30
@@ -163,11 +162,8 @@ class SnowparkAnnotationProcessor(ArtifactProcessor):
     and generate SQL code for creation of extension functions based on those discovered objects.
     """
 
-    def __init__(
-        self,
-        na_project: NativeAppProjectModel,
-    ):
-        super().__init__(na_project=na_project)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
 
     def process(
         self,
@@ -181,8 +177,8 @@ def process(
         """
 
         bundle_map = BundleMap(
-            project_root=self._na_project.project_root,
-            deploy_root=self._na_project.deploy_root,
+            project_root=self._bundle_ctx.project_root,
+            deploy_root=self._bundle_ctx.deploy_root,
         )
         bundle_map.add(artifact_to_process)
 
@@ -235,7 +231,7 @@ def process(
 
     @property
     def _generated_root(self):
-        return self._na_project.generated_root / "snowpark"
+        return self._bundle_ctx.generated_root / "snowpark"
 
     def _normalize_imports(
         self,
@@ -315,7 +311,7 @@ def collect_extension_functions(
         self, bundle_map: BundleMap, processor_mapping: Optional[ProcessorMapping]
     ) -> Dict[Path, List[NativeAppExtensionFunction]]:
         kwargs = (
-            _determine_virtual_env(self._na_project.project_root, processor_mapping)
+            _determine_virtual_env(self._bundle_ctx.project_root, processor_mapping)
             if processor_mapping is not None
             else {}
         )
@@ -338,7 +334,7 @@ def collect_extension_functions(
             )
             collected_extension_function_json = _execute_in_sandbox(
                 py_file=str(dest_file.resolve()),
-                deploy_root=self._na_project.deploy_root,
+                deploy_root=self._bundle_ctx.deploy_root,
                 kwargs=kwargs,
             )
 
@@ -369,7 +365,7 @@ def generate_new_sql_file_name(self, py_file: Path) -> Path:
         """
         Generates a SQL filename for the generated root from the python file, and creates its parent directories.
         """
-        relative_py_file = py_file.relative_to(self._na_project.deploy_root)
+        relative_py_file = py_file.relative_to(self._bundle_ctx.deploy_root)
         sql_file = Path(self._generated_root, relative_py_file.with_suffix(".sql"))
         if sql_file.exists():
             cc.warning(

src/snowflake/cli/plugins/nativeapp/manager.py

@@ -350,9 +350,7 @@ def build_bundle(self) -> BundleMap:
         Populates the local deploy root from artifact sources.
         """
         bundle_map = build_bundle(self.project_root, self.deploy_root, self.artifacts)
-        compiler = NativeAppCompiler(
-            na_project=self.na_project,
-        )
+        compiler = NativeAppCompiler(self.na_project.get_bundle_context())
         compiler.compile_artifacts()
         return bundle_map
 

src/snowflake/cli/plugins/nativeapp/project_model.py

@@ -31,6 +31,7 @@
 from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping
 from snowflake.cli.api.project.util import extract_schema, to_identifier
 from snowflake.cli.plugins.nativeapp.artifacts import resolve_without_follow
+from snowflake.cli.plugins.nativeapp.bundle_context import BundleContext
 from snowflake.connector import DictCursor
 
 
@@ -185,3 +186,13 @@ def debug_mode(self) -> Optional[bool]:
         if self.definition.application:
             return self.definition.application.debug
         return None
+
+    def get_bundle_context(self) -> BundleContext:
+        return BundleContext(
+            package_name=self.package_name,
+            artifacts=self.artifacts,
+            project_root=self.project_root,
+            bundle_root=self.bundle_root,
+            deploy_root=self.deploy_root,
+            generated_root=self.generated_root,
+        )

tests/nativeapp/codegen/snowpark/test_python_processor.py

@@ -353,7 +353,8 @@ def test_process_no_collected_functions(
                 project_definition=native_app_project_instance.native_app,
                 project_root=local_path,
             )
-            SnowparkAnnotationProcessor(na_project=project).process(
+            processor = SnowparkAnnotationProcessor(project.get_bundle_context())
+            processor.process(
                 artifact_to_process=native_app_project_instance.native_app.artifacts[0],
                 processor_mapping=ProcessorMapping(name="SNOWPARK"),
                 write_to_sql=False,  # For testing
@@ -404,7 +405,8 @@ def test_process_with_collected_functions(
                 project_definition=native_app_project_instance.native_app,
                 project_root=local_path,
             )
-            SnowparkAnnotationProcessor(na_project=project).process(
+            processor = SnowparkAnnotationProcessor(project.get_bundle_context())
+            processor.process(
                 artifact_to_process=native_app_project_instance.native_app.artifacts[0],
                 processor_mapping=processor_mapping,
             )
@@ -463,7 +465,8 @@ def test_package_normalization(
                 project_definition=native_app_project_instance.native_app,
                 project_root=local_path,
             )
-            SnowparkAnnotationProcessor(na_project=project).process(
+            processor = SnowparkAnnotationProcessor(project.get_bundle_context())
+            processor.process(
                 artifact_to_process=native_app_project_instance.native_app.artifacts[0],
                 processor_mapping=processor_mapping,
             )

tests/nativeapp/codegen/test_compiler.py

@@ -58,9 +58,8 @@ def test_proj_def():
 
 @pytest.fixture()
 def test_compiler(test_proj_def):
-    return NativeAppCompiler(
-        na_project=create_native_app_project_model(test_proj_def.native_app)
-    )
+    na_project = create_native_app_project_model(test_proj_def.native_app)
+    return NativeAppCompiler(na_project.get_bundle_context())
 
 
 def test_try_create_processor_returns_none(test_proj_def, test_compiler):
@@ -92,7 +91,7 @@ def test_find_and_execute_processors_exception(test_proj_def, test_compiler):
     app_pkg = create_native_app_project_model(
         project_definition=test_proj_def.native_app
     )
-    test_compiler = NativeAppCompiler(na_project=app_pkg)
+    test_compiler = NativeAppCompiler(app_pkg.get_bundle_context())
 
     with pytest.raises(UnsupportedArtifactProcessorError):
         test_compiler.compile_artifacts()

tests/nativeapp/test_artifacts.py

@@ -34,7 +34,10 @@
     symlink_or_copy,
 )
 
-from tests.nativeapp.utils import assert_dir_snapshot, touch
+from tests.nativeapp.utils import (
+    assert_dir_snapshot,
+    touch,
+)
 from tests.testing_utils.files_and_dirs import pushd, temp_local_dir
 from tests_common import IS_WINDOWS