Moved Snowpark and Streamlit artifacts to separate directory in outpu… (

#1909)

Moved Snowpark and Streamlit artifacts to separate directory in output/deploy

src/snowflake/cli/_plugins/snowpark/commands.py

@@ -220,19 +220,19 @@ def build_artifacts_mappings(
 ) -> Tuple[EntityToImportPathsMapping, StageToArtefactMapping]:
     stages_to_artifact_map: StageToArtefactMapping = defaultdict(set)
     entities_to_imports_map: EntityToImportPathsMapping = defaultdict(set)
-    for entity_id, entity in snowpark_entities.items():
+    for name, entity in snowpark_entities.items():
         stage = entity.stage
         required_artifacts = set()
         for artefact in entity.artifacts:
             artefact_dto = project_paths.get_artefact_dto(artefact)
             required_artifacts.add(artefact_dto)
-            entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage))
+            entities_to_imports_map[name].add(artefact_dto.import_path(stage))
         stages_to_artifact_map[stage].update(required_artifacts)
 
         deps_artefact = project_paths.get_dependencies_artefact()
         if deps_artefact.post_build_path.exists():
             stages_to_artifact_map[stage].add(deps_artefact)
-            entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage))
+            entities_to_imports_map[name].add(deps_artefact.import_path(stage))
     return entities_to_imports_map, stages_to_artifact_map
 
 
@@ -330,7 +330,7 @@ def build(
     anaconda_packages_manager = AnacondaPackagesManager()
 
     # Clean up deploy root
-    project_paths.remove_up_deploy_root()
+    project_paths.remove_up_bundle_root()
 
     # Resolve dependencies
     if project_paths.requirements.exists():
@@ -389,7 +389,7 @@ def build(
             for artefact in artifacts:
                 bundle_map = BundleMap(
                     project_root=artefact.project_root,
-                    deploy_root=project_paths.deploy_root,
+                    deploy_root=project_paths.bundle_root,
                 )
                 bundle_map.add(PathMapping(src=str(artefact.path), dest=artefact.dest))
 

src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py

@@ -25,7 +25,7 @@
 from snowflake.cli.api.constants import DEPLOYMENT_STAGE
 from snowflake.cli.api.feature_flags import FeatureFlag
 from snowflake.cli.api.identifiers import FQN
-from snowflake.cli.api.project.project_paths import ProjectPaths
+from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root
 from snowflake.cli.api.project.schemas.entities.common import PathMapping
 from snowflake.cli.api.secure_path import SecurePath
 
@@ -45,6 +45,7 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact:
         if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
             return Artefact(
                 project_root=self.project_root,
+                bundle_root=self.bundle_root,
                 dest=artifact_path.dest,
                 path=Path(artifact_path.src),
             )
@@ -57,7 +58,10 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact:
     def get_dependencies_artefact(self) -> Artefact:
         if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
             return Artefact(
-                project_root=self.project_root, dest=None, path=Path("dependencies.zip")
+                project_root=self.project_root,
+                bundle_root=self.bundle_root,
+                dest=None,
+                path=Path("dependencies.zip"),
             )
         else:
             return ArtefactOldBuild(
@@ -74,19 +78,29 @@ def snowflake_requirements(self) -> SecurePath:
     def requirements(self) -> SecurePath:
         return SecurePath(self.path_relative_to_root(Path("requirements.txt")))
 
+    @property
+    def bundle_root(self) -> Path:
+        return bundle_root(self.project_root, "snowpark")
+
 
 @dataclass(unsafe_hash=True)
 class Artefact:
     """Helper for getting paths related to given artefact."""
 
     project_root: Path
+    bundle_root: Path
     path: Path
     dest: str | None = None
 
     def __init__(
-        self, project_root: Path, path: Path, dest: Optional[str] = None
+        self,
+        project_root: Path,
+        bundle_root: Path,
+        path: Path,
+        dest: Optional[str] = None,
     ) -> None:
         self.project_root = project_root
+        self.bundle_root = bundle_root
         self.path = path
         self.dest = dest
         if self.dest and not self._is_dest_a_file() and not self.dest.endswith("/"):
@@ -117,15 +131,15 @@ def post_build_path(self) -> Path:
         """
         Returns post-build artefact path. Directories are mapped to corresponding .zip files.
         """
-        deploy_root = self.deploy_root()
+        bundle_root = self.bundle_root
         path = (
             self._path_until_asterisk()
             if glob.has_magic(str(self.path))
             else self.path.parent
         )
         if self._is_dest_a_file():
-            return deploy_root / self.dest  # type: ignore
-        return deploy_root / (self.dest or path) / self._artefact_name
+            return bundle_root / self.dest  # type: ignore
+        return bundle_root / (self.dest or path) / self._artefact_name
 
     def upload_path(self, stage: FQN | str | None) -> str:
         """
@@ -153,9 +167,6 @@ def import_path(self, stage: FQN | str | None) -> str:
         """Path for UDF/sproc imports clause."""
         return self.upload_path(stage) + self._artefact_name
 
-    def deploy_root(self) -> Path:
-        return self.project_root / "output"
-
     def _is_dest_a_file(self) -> bool:
         if not self.dest:
             return False
@@ -183,7 +194,7 @@ class ArtefactOldBuild(Artefact):
     dest: str | None = None
 
     def __init__(self, path: Path, dest: Optional[str] = None) -> None:
-        super().__init__(project_root=Path(), path=path, dest=dest)
+        super().__init__(project_root=Path(), bundle_root=Path(), path=path, dest=dest)
 
     @property
     def _artefact_name(self) -> str:

src/snowflake/cli/_plugins/streamlit/manager.py

@@ -68,29 +68,31 @@ def _put_streamlit_files(
         if not artifacts:
             return
         stage_manager = StageManager()
+        # We treat the bundle root as deploy root
         bundle_map = BundleMap(
             project_root=streamlit_project_paths.project_root,
-            deploy_root=streamlit_project_paths.deploy_root,
+            deploy_root=streamlit_project_paths.bundle_root,
         )
         for artifact in artifacts:
             bundle_map.add(PathMapping(src=str(artifact.src), dest=artifact.dest))
 
         # Clean up deploy root
-        streamlit_project_paths.remove_up_deploy_root()
+        streamlit_project_paths.remove_up_bundle_root()
 
         for (absolute_src, absolute_dest) in bundle_map.all_mappings(
             absolute=True, expand_directories=True
         ):
             if absolute_src.is_file():
+                # We treat the bundle root as deploy root
                 symlink_or_copy(
                     absolute_src,
                     absolute_dest,
-                    deploy_root=streamlit_project_paths.deploy_root,
+                    deploy_root=streamlit_project_paths.bundle_root,
                 )
                 # Temporary solution, will be replaced with diff
                 stage_path = (
                     PurePosixPath(absolute_dest)
-                    .relative_to(streamlit_project_paths.deploy_root)
+                    .relative_to(streamlit_project_paths.bundle_root)
                     .parent
                 )
                 full_stage_path = f"{stage_root}/{stage_path}".rstrip("/")

src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py

@@ -14,12 +14,17 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from pathlib import Path
 
-from snowflake.cli.api.project.project_paths import ProjectPaths
+from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root
 
 
 @dataclass
 class StreamlitProjectPaths(ProjectPaths):
     """
     This class allows you to manage files paths related to given project.
     """
+
+    @property
+    def bundle_root(self) -> Path:
+        return bundle_root(self.project_root, "streamlit")

src/snowflake/cli/api/project/project_paths.py

@@ -8,9 +8,15 @@ class ProjectPaths:
     project_root: Path
 
     @property
-    def deploy_root(self) -> Path:
-        return self.project_root / "output"
+    def bundle_root(self) -> Path:
+        return bundle_root(self.project_root)
 
-    def remove_up_deploy_root(self) -> None:
-        if self.deploy_root.exists():
-            rmtree(self.deploy_root)
+    def remove_up_bundle_root(self) -> None:
+        if self.bundle_root.exists():
+            rmtree(self.bundle_root)
+
+
+def bundle_root(root: Path, app_type: str | None = None) -> Path:
+    if app_type:
+        return root / "output" / "bundle" / app_type
+    return root / "output" / "bundle"

tests/snowpark/test_artifacts.py

@@ -12,42 +12,44 @@
     lambda _: True,
 )
 
+bundle_root = Path("output") / "bundle" / "snowpark"
+
 
 @pytest.mark.parametrize(
     "artifacts, local_path, stage_path",
     [
-        ("src", Path("output") / "src.zip", "/"),
-        ("src/", Path("output") / "src.zip", "/"),
-        ("src/*", Path("output") / "src.zip", "/"),
-        ("src/*.py", Path("output") / "src.zip", "/"),
+        ("src", bundle_root / "src.zip", "/"),
+        ("src/", bundle_root / "src.zip", "/"),
+        ("src/*", bundle_root / "src.zip", "/"),
+        ("src/*.py", bundle_root / "src.zip", "/"),
         (
             "src/dir/dir_app.py",
-            Path("output") / "src" / "dir" / "dir_app.py",
+            bundle_root / "src" / "dir" / "dir_app.py",
             "/src/dir/",
         ),
         (
             {"src": "src/**/*", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src/", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src/*", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"},
-            "output/source/dir/apps/dir_app.py",
+            bundle_root / "source" / "dir" / "apps" / "dir_app.py",
             "/source/dir/apps/",
         ),
     ],
@@ -111,38 +113,38 @@ def test_build_and_deploy_with_artifacts(
 @pytest.mark.parametrize(
     "artifact, local_path, stage_path",
     [
-        ("src", Path("output") / "src.zip", "/"),
-        ("src/", Path("output") / "src.zip", "/"),
-        ("src/*", Path("output") / "src.zip", "/"),
-        ("src/*.py", Path("output") / "src.zip", "/"),
+        ("src", bundle_root / "src.zip", "/"),
+        ("src/", bundle_root / "src.zip", "/"),
+        ("src/*", bundle_root / "src.zip", "/"),
+        ("src/*.py", bundle_root / "src.zip", "/"),
         (
             "src/dir/dir_app.py",
-            Path("output") / "src" / "dir" / "dir_app.py",
+            bundle_root / "src" / "dir" / "dir_app.py",
             "/src/dir/",
         ),
         (
             {"src": "src/**/*", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src/", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src/*", "dest": "source/"},
-            Path("output") / "source" / "src.zip",
+            bundle_root / "source" / "src.zip",
             "/source/",
         ),
         (
             {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"},
-            Path("output") / "source" / "dir" / "apps" / "dir_app.py",
+            bundle_root / "source" / "dir" / "apps" / "dir_app.py",
             "/source/dir/apps/",
         ),
     ],

tests/snowpark/test_build.py

@@ -58,7 +58,9 @@ def test_build_with_glob_patterns_in_artifacts(
 
         result = runner.invoke(["snowpark", "build", "--ignore-anaconda"])
         assert result.exit_code == 0, result.output
-        _assert_zip_contains(tmp_dir / "output" / zip_name, expected_files)
+        _assert_zip_contains(
+            tmp_dir / "output" / "bundle" / "snowpark" / zip_name, expected_files
+        )
 
 
 def _assert_zip_contains(app_zip: str, expected_files: Set[str]):

tests/snowpark/test_function.py

@@ -236,7 +236,11 @@ def test_deploy_function_no_changes(
     ]
     assert queries == [
         "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'",
-        f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True",
+        _put_query(
+            Path(project_dir),
+            "my_snowpark_project/app.py",
+            "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/",
+        ),
     ]
 
 
@@ -286,7 +290,11 @@ def test_deploy_function_needs_update_because_packages_changes(
     ]
     assert queries == [
         "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'",
-        f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True",
+        _put_query(
+            Path(project_dir),
+            "my_snowpark_project/app.py",
+            "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/",
+        ),
         dedent(
             """\
             create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant)
@@ -348,8 +356,11 @@ def test_deploy_function_needs_update_because_handler_changes(
     ]
     assert queries == [
         "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'",
-        f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/"
-        f" auto_compress=false parallel=4 overwrite=True",
+        _put_query(
+            Path(project_dir),
+            "my_snowpark_project/app.py",
+            "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/",
+        ),
         dedent(
             """\
             create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant)
@@ -712,3 +723,9 @@ def test_command_aliases(mock_connector, runner, mock_ctx, command, parameters):
 
     queries = ctx.get_queries()
     assert queries[0] == queries[1]
+
+
+def _put_query(project_root: Path, source: str, dest: str):
+    return dedent(
+        f"put file://{project_root.resolve() / 'output' / 'bundle' / 'snowpark' / source} {dest} auto_compress=false parallel=4 overwrite=True"
+    )