From 7735878d61252d0f03c53ef2b5088085210e7361 Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Mon, 2 Dec 2024 15:19:08 +0100 Subject: [PATCH] Moved Snowpark and Streamlit artifacts to separate directory in output/deploy --- .../cli/_plugins/snowpark/commands.py | 6 +- .../snowpark/snowpark_project_paths.py | 25 +++-- .../streamlit/streamlit_project_paths.py | 5 + .../cli/api/project/project_paths.py | 2 +- tests/snowpark/test_artifacts.py | 42 ++++---- tests/snowpark/test_build.py | 4 +- tests/snowpark/test_function.py | 6 +- tests/snowpark/test_procedure.py | 4 +- tests/snowpark/test_project_paths.py | 81 ++++++++------ tests/streamlit/test_artifacts.py | 100 +++++++++++------- tests/streamlit/test_commands.py | 2 +- tests_integration/test_snowpark.py | 25 +++-- .../testing_utils/snowpark_utils.py | 4 +- 13 files changed, 192 insertions(+), 114 deletions(-) diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py index 28db01d068..dab880149f 100644 --- a/src/snowflake/cli/_plugins/snowpark/commands.py +++ b/src/snowflake/cli/_plugins/snowpark/commands.py @@ -220,19 +220,19 @@ def build_artifacts_mappings( ) -> Tuple[EntityToImportPathsMapping, StageToArtefactMapping]: stages_to_artifact_map: StageToArtefactMapping = defaultdict(set) entities_to_imports_map: EntityToImportPathsMapping = defaultdict(set) - for entity_id, entity in snowpark_entities.items(): + for name, entity in snowpark_entities.items(): stage = entity.stage required_artifacts = set() for artefact in entity.artifacts: artefact_dto = project_paths.get_artefact_dto(artefact) required_artifacts.add(artefact_dto) - entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage)) + entities_to_imports_map[name].add(artefact_dto.import_path(stage)) stages_to_artifact_map[stage].update(required_artifacts) deps_artefact = project_paths.get_dependencies_artefact() if deps_artefact.post_build_path.exists(): stages_to_artifact_map[stage].add(deps_artefact) - entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage)) + entities_to_imports_map[name].add(deps_artefact.import_path(stage)) return entities_to_imports_map, stages_to_artifact_map diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py index 6e103d840e..7703022a4a 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py @@ -45,6 +45,7 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): return Artefact( project_root=self.project_root, + deploy_root=self.deploy_root, dest=artifact_path.dest, path=Path(artifact_path.src), ) @@ -57,7 +58,10 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: def get_dependencies_artefact(self) -> Artefact: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): return Artefact( - project_root=self.project_root, dest=None, path=Path("dependencies.zip") + project_root=self.project_root, + deploy_root=self.deploy_root, + dest=None, + path=Path("dependencies.zip"), ) else: return ArtefactOldBuild( @@ -74,19 +78,29 @@ def snowflake_requirements(self) -> SecurePath: def requirements(self) -> SecurePath: return SecurePath(self.path_relative_to_root(Path("requirements.txt"))) + @property + def deploy_root(self) -> Path: + return self.project_root / "output" / "deploy" / "snowpark" + @dataclass(unsafe_hash=True) class Artefact: """Helper for getting paths related to given artefact.""" project_root: Path + deploy_root: Path path: Path dest: str | None = None def __init__( - self, project_root: Path, path: Path, dest: Optional[str] = None + self, + project_root: Path, + deploy_root: Path, + path: Path, + dest: Optional[str] = None, ) -> None: self.project_root = project_root + self.deploy_root = deploy_root self.path = path self.dest = dest if self.dest and not self._is_dest_a_file() and not self.dest.endswith("/"): @@ -117,7 +131,7 @@ def post_build_path(self) -> Path: """ Returns post-build artefact path. Directories are mapped to corresponding .zip files. """ - deploy_root = self.deploy_root() + deploy_root = self.deploy_root path = ( self._path_until_asterisk() if glob.has_magic(str(self.path)) @@ -153,9 +167,6 @@ def import_path(self, stage: FQN | str | None) -> str: """Path for UDF/sproc imports clause.""" return self.upload_path(stage) + self._artefact_name - def deploy_root(self) -> Path: - return self.project_root / "output" - def _is_dest_a_file(self) -> bool: if not self.dest: return False @@ -183,7 +194,7 @@ class ArtefactOldBuild(Artefact): dest: str | None = None def __init__(self, path: Path, dest: Optional[str] = None) -> None: - super().__init__(project_root=Path(), path=path, dest=dest) + super().__init__(project_root=Path(), deploy_root=Path(), path=path, dest=dest) @property def _artefact_name(self) -> str: diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py index fbe89959b2..e1642f53f2 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py @@ -14,6 +14,7 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from snowflake.cli.api.project.project_paths import ProjectPaths @@ -23,3 +24,7 @@ class StreamlitProjectPaths(ProjectPaths): """ This class allows you to manage files paths related to given project. """ + + @property + def deploy_root(self) -> Path: + return self.project_root / "output" / "deploy" / "streamlit" diff --git a/src/snowflake/cli/api/project/project_paths.py b/src/snowflake/cli/api/project/project_paths.py index f6b2a14d99..8d67d212cc 100644 --- a/src/snowflake/cli/api/project/project_paths.py +++ b/src/snowflake/cli/api/project/project_paths.py @@ -9,7 +9,7 @@ class ProjectPaths: @property def deploy_root(self) -> Path: - return self.project_root / "output" + return self.project_root / "output" / "deploy" def remove_up_deploy_root(self) -> None: if self.deploy_root.exists(): diff --git a/tests/snowpark/test_artifacts.py b/tests/snowpark/test_artifacts.py index c573ab9b6d..efe55a08bc 100644 --- a/tests/snowpark/test_artifacts.py +++ b/tests/snowpark/test_artifacts.py @@ -12,42 +12,44 @@ lambda _: True, ) +deploy_root = Path("output") / "deploy" / "snowpark" + @pytest.mark.parametrize( "artifacts, local_path, stage_path", [ - ("src", Path("output") / "src.zip", "/"), - ("src/", Path("output") / "src.zip", "/"), - ("src/*", Path("output") / "src.zip", "/"), - ("src/*.py", Path("output") / "src.zip", "/"), + ("src", deploy_root / "src.zip", "/"), + ("src/", deploy_root / "src.zip", "/"), + ("src/*", deploy_root / "src.zip", "/"), + ("src/*.py", deploy_root / "src.zip", "/"), ( "src/dir/dir_app.py", - Path("output") / "src" / "dir" / "dir_app.py", + deploy_root / "src" / "dir" / "dir_app.py", "/src/dir/", ), ( {"src": "src/**/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src/", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, - "output/source/dir/apps/dir_app.py", + deploy_root / "source" / "dir" / "apps" / "dir_app.py", "/source/dir/apps/", ), ], @@ -111,38 +113,38 @@ def test_build_and_deploy_with_artifacts( @pytest.mark.parametrize( "artifact, local_path, stage_path", [ - ("src", Path("output") / "src.zip", "/"), - ("src/", Path("output") / "src.zip", "/"), - ("src/*", Path("output") / "src.zip", "/"), - ("src/*.py", Path("output") / "src.zip", "/"), + ("src", deploy_root / "src.zip", "/"), + ("src/", deploy_root / "src.zip", "/"), + ("src/*", deploy_root / "src.zip", "/"), + ("src/*.py", deploy_root / "src.zip", "/"), ( "src/dir/dir_app.py", - Path("output") / "src" / "dir" / "dir_app.py", + deploy_root / "src" / "dir" / "dir_app.py", "/src/dir/", ), ( {"src": "src/**/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src/", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + deploy_root / "source" / "src.zip", "/source/", ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, - Path("output") / "source" / "dir" / "apps" / "dir_app.py", + deploy_root / "source" / "dir" / "apps" / "dir_app.py", "/source/dir/apps/", ), ], diff --git a/tests/snowpark/test_build.py b/tests/snowpark/test_build.py index 9298ed2b32..cefc902fb2 100644 --- a/tests/snowpark/test_build.py +++ b/tests/snowpark/test_build.py @@ -58,7 +58,9 @@ def test_build_with_glob_patterns_in_artifacts( result = runner.invoke(["snowpark", "build", "--ignore-anaconda"]) assert result.exit_code == 0, result.output - _assert_zip_contains(tmp_dir / "output" / zip_name, expected_files) + _assert_zip_contains( + tmp_dir / "output" / "deploy" / "snowpark" / zip_name, expected_files + ) def _assert_zip_contains(app_zip: str, expected_files: Set[str]): diff --git a/tests/snowpark/test_function.py b/tests/snowpark/test_function.py index 7ee4a27072..2fe20ac539 100644 --- a/tests/snowpark/test_function.py +++ b/tests/snowpark/test_function.py @@ -214,7 +214,7 @@ def test_deploy_function_no_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + f"put file://{Path(project_dir).resolve()}/output/deploy/snowpark/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", ] @@ -259,7 +259,7 @@ def test_deploy_function_needs_update_because_packages_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + f"put file://{Path(project_dir).resolve()}/output/deploy/snowpark/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", dedent( """\ create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) @@ -316,7 +316,7 @@ def test_deploy_function_needs_update_because_handler_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f"put file://{Path(project_dir).resolve()}/output/deploy/snowpark/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" f" auto_compress=false parallel=4 overwrite=True", dedent( """\ diff --git a/tests/snowpark/test_procedure.py b/tests/snowpark/test_procedure.py index fc13e1e6ef..c1c6d7de24 100644 --- a/tests/snowpark/test_procedure.py +++ b/tests/snowpark/test_procedure.py @@ -105,7 +105,7 @@ def test_deploy_procedure( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(tmp).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + f"put file://{Path(tmp).resolve()}/output/deploy/snowpark/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", dedent( """\ create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) @@ -190,7 +190,7 @@ def test_deploy_procedure_with_external_access( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f"put file://{Path(project_dir).resolve()}/output/deploy/snowpark/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" f" auto_compress=false parallel=4 overwrite=True", dedent( """\ diff --git a/tests/snowpark/test_project_paths.py b/tests/snowpark/test_project_paths.py index 55796e1a98..b2e425accc 100644 --- a/tests/snowpark/test_project_paths.py +++ b/tests/snowpark/test_project_paths.py @@ -4,6 +4,9 @@ import pytest from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact +deploy_path = Path("output") / "deploy" / "snowpark" +absolute_deploy_path = Path.cwd().absolute() / "output" / "deploy" / "snowpark" + @pytest.mark.parametrize( "path, dest, is_file, expected_path", @@ -31,7 +34,9 @@ def test_artifact_import_path(mock_ctx_context, path, dest, is_file, expected_pa stage = "stage" with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - import_path = Artefact(Path(), Path(path), dest).import_path(stage) + import_path = Artefact( + Path(), Path("output") / "deploy" / "snowpark", Path(path), dest + ).import_path(stage) assert import_path == expected_path @@ -61,7 +66,9 @@ def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_pa mock_ctx_context.return_value.connection = mock_connection with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - upload_path = Artefact(Path(), Path(path), dest).upload_path("stage") + upload_path = Artefact( + Path(), Path("output") / "deploy" / "snowpark", Path(path), dest + ).upload_path("stage") assert upload_path == expected_path @@ -69,39 +76,41 @@ def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_pa @pytest.mark.parametrize( "path, dest, is_file, expected_path", [ - ("src", None, False, Path("output") / "src.zip"), - ("src/", None, False, Path("output") / "src.zip"), - ("src", "source", False, Path("output") / "source" / "src.zip"), - ("src/app.py", None, True, Path("output") / "src" / "app.py"), + ("src", None, False, deploy_path / "src.zip"), + ("src/", None, False, deploy_path / "src.zip"), + ("src", "source", False, deploy_path / "source" / "src.zip"), + ("src/app.py", None, True, deploy_path / "src" / "app.py"), ( "src/app.py", "source/new_app.py", True, - Path("output") / "source" / "new_app.py", + deploy_path / "source" / "new_app.py", ), - ("src/*", "source/new_app.py", True, Path("output") / "source" / "new_app.py"), + ("src/*", "source/new_app.py", True, deploy_path / "source" / "new_app.py"), ( "src/dir/dir2/app.py", None, True, - Path("output") / "src" / "dir" / "dir2" / "app.py", + deploy_path / "src" / "dir" / "dir2" / "app.py", ), ( "src/dir/dir2/app.py", "source/", True, - Path("output") / "source" / "app.py", + deploy_path / "source" / "app.py", ), - ("src/*", "source/", False, Path("output") / "source" / "src.zip"), - ("src/**/*.py", None, False, Path("output") / "src.zip"), - ("src/**/*.py", "source/", False, Path("output") / "source" / "src.zip"), - ("src/app*", None, False, Path("output") / "src.zip"), - ("src/app[1-5].py", None, False, Path("output") / "src.zip"), + ("src/*", "source/", False, deploy_path / "source" / "src.zip"), + ("src/**/*.py", None, False, deploy_path / "src.zip"), + ("src/**/*.py", "source/", False, deploy_path / "source" / "src.zip"), + ("src/app*", None, False, deploy_path / "src.zip"), + ("src/app[1-5].py", None, False, deploy_path / "src.zip"), ], ) def test_artifact_post_build_path(path, dest, is_file, expected_path): with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - post_build_path = Artefact(Path(), Path(path), dest).post_build_path + post_build_path = Artefact( + Path(), Path("output") / "deploy" / "snowpark", Path(path), dest + ).post_build_path assert post_build_path == expected_path @@ -134,7 +143,12 @@ def test_artifact_import_path_from_other_directory( stage = "stage" with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - import_path = Artefact(Path("/tmp"), Path(path), dest).import_path(stage) + import_path = Artefact( + Path("/tmp"), + Path("/tmp") / "output" / "deploy" / "snowpark", + Path(path), + dest, + ).import_path(stage) assert import_path == expected_path @@ -166,7 +180,9 @@ def test_artifact_upload_path_from_other_directory( mock_ctx_context.return_value.connection = mock_connection with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - upload_path = Artefact(Path("/tmp"), Path(path), dest).upload_path("stage") + upload_path = Artefact( + Path("/tmp"), Path("/tmp") / "output" / "deploy", Path(path), dest + ).upload_path("stage") assert upload_path == expected_path @@ -174,48 +190,48 @@ def test_artifact_upload_path_from_other_directory( @pytest.mark.parametrize( "path, dest, is_file, expected_path", [ - ("src", None, False, Path.cwd().absolute() / "output" / "src.zip"), - ("src/", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src", None, False, absolute_deploy_path / "src.zip"), + ("src/", None, False, absolute_deploy_path / "src.zip"), ( "src", "source", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_deploy_path / "source" / "src.zip", ), - ("src/app.py", None, True, Path.cwd().absolute() / "output" / "src" / "app.py"), + ("src/app.py", None, True, absolute_deploy_path / "src" / "app.py"), ( "src/app.py", "source/new_app.py", True, - Path.cwd().absolute() / "output" / "source" / "new_app.py", + absolute_deploy_path / "source" / "new_app.py", ), ( "src/dir/dir2/app.py", None, True, - Path.cwd().absolute() / "output" / "src" / "dir" / "dir2" / "app.py", + absolute_deploy_path / "src" / "dir" / "dir2" / "app.py", ), ( "src/dir/dir2/app.py", "source/", True, - Path.cwd().absolute() / "output" / "source" / "app.py", + absolute_deploy_path / "source" / "app.py", ), ( "src/*", "source/", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_deploy_path / "source" / "src.zip", ), - ("src/**/*.py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/**/*.py", None, False, absolute_deploy_path / "src.zip"), ( "src/**/*.py", "source/", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_deploy_path / "source" / "src.zip", ), - ("src/app*", None, False, Path.cwd().absolute() / "output" / "src.zip"), - ("src/app[1-5].py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/app*", None, False, absolute_deploy_path / "src.zip"), + ("src/app[1-5].py", None, False, absolute_deploy_path / "src.zip"), ], ) def test_artifact_post_build_path_from_other_directory( @@ -223,7 +239,10 @@ def test_artifact_post_build_path_from_other_directory( ): with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): post_build_path = Artefact( - Path.cwd().absolute(), Path(path), dest + Path.cwd().absolute(), + Path.cwd().absolute() / "output" / "deploy" / "snowpark", + Path(path), + dest, ).post_build_path assert post_build_path == expected_path diff --git a/tests/streamlit/test_artifacts.py b/tests/streamlit/test_artifacts.py index 4b75406d1c..ff0acd0cb5 100644 --- a/tests/streamlit/test_artifacts.py +++ b/tests/streamlit/test_artifacts.py @@ -6,6 +6,8 @@ from snowflake.cli._plugins.connection.util import UIParameter from snowflake.connector.compat import IS_WINDOWS +deploy_root = Path("output") / "deploy" / "streamlit" + @pytest.mark.parametrize( "artifacts, paths", @@ -13,9 +15,9 @@ ( "src", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": deploy_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": deploy_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], @@ -23,9 +25,9 @@ ( "src/", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": deploy_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": deploy_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], @@ -33,19 +35,19 @@ ( "src/*", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": deploy_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": deploy_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], ), - ("src/*.py", [{"local": Path("output") / "src" / "app.py", "stage": "/src"}]), + ("src/*.py", [{"local": deploy_root / "src" / "app.py", "stage": "/src"}]), ( "src/dir/dir_app.py", [ { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": deploy_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", } ], @@ -53,10 +55,10 @@ ( {"src": "src/**/*", "dest": "source/"}, [ - {"local": Path("output") / "source" / "app.py", "stage": "/source"}, - {"local": Path("output") / "source" / "dir_app.py", "stage": "/source"}, + {"local": deploy_root / "source" / "app.py", "stage": "/source"}, + {"local": deploy_root / "source" / "dir_app.py", "stage": "/source"}, { - "local": Path("output") / "source" / "dir" / "dir_app.py", + "local": deploy_root / "source" / "dir" / "dir_app.py", "stage": "/source/dir", }, ], @@ -65,11 +67,11 @@ {"src": "src", "dest": "source/"}, [ { - "local": Path("output") / "source" / "src" / "app.py", + "local": deploy_root / "source" / "src" / "app.py", "stage": "/source/src", }, { - "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "local": deploy_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -78,11 +80,11 @@ {"src": "src/", "dest": "source/"}, [ { - "local": Path("output") / "source" / "src" / "app.py", + "local": deploy_root / "source" / "src" / "app.py", "stage": "/source/src", }, { - "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "local": deploy_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -90,9 +92,9 @@ ( {"src": "src/*", "dest": "source/"}, [ - {"local": Path("output") / "source" / "app.py", "stage": "/source"}, + {"local": deploy_root / "source" / "app.py", "stage": "/source"}, { - "local": Path("output") / "source" / "dir" / "dir_app.py", + "local": deploy_root / "source" / "dir" / "dir_app.py", "stage": "/source/dir", }, ], @@ -101,7 +103,7 @@ {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, [ { - "local": Path("output") / "source" / "dir" / "apps" / "dir_app.py", + "local": deploy_root / "source" / "dir" / "apps" / "dir_app.py", "stage": "/source/dir/apps", } ], @@ -179,43 +181,63 @@ def test_deploy_with_artifacts( ( "src", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": deploy_root / "src" / "app.py", "stage": "/src"}, + { + "local": deploy_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), ( "src/", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": deploy_root / "src" / "app.py", "stage": "/src"}, + { + "local": deploy_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), ( "src/*", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": deploy_root / "src" / "app.py", "stage": "/src"}, + { + "local": deploy_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), - ("src/*.py", [{"local": "output/src/app.py", "stage": "/src"}]), + ("src/*.py", [{"local": deploy_root / "src" / "app.py", "stage": "/src"}]), ( "src/dir/dir_app.py", - [{"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}], + [ + { + "local": deploy_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + } + ], ), ( {"src": "src/**/*", "dest": "source/"}, [ - {"local": "output/source/app.py", "stage": "/source"}, - {"local": "output/source/dir_app.py", "stage": "/source"}, - {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + {"local": deploy_root / "source" / "app.py", "stage": "/source"}, + {"local": deploy_root / "source" / "dir_app.py", "stage": "/source"}, + { + "local": deploy_root / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, ], ), ( {"src": "src", "dest": "source/"}, [ - {"local": "output/source/src/app.py", "stage": "/source/src"}, { - "local": "output/source/src/dir/dir_app.py", + "local": deploy_root / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": deploy_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -223,9 +245,12 @@ def test_deploy_with_artifacts( ( {"src": "src/", "dest": "source/"}, [ - {"local": "output/source/src/app.py", "stage": "/source/src"}, { - "local": "output/source/src/dir/dir_app.py", + "local": deploy_root / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": deploy_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -233,15 +258,18 @@ def test_deploy_with_artifacts( ( {"src": "src/*", "dest": "source/"}, [ - {"local": "output/source/app.py", "stage": "/source"}, - {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + {"local": deploy_root / "source" / "app.py", "stage": "/source"}, + { + "local": deploy_root / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, ], ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, [ { - "local": "output/source/dir/apps/dir_app.py", + "local": deploy_root / "source" / "dir" / "apps" / "dir_app.py", "stage": "/source/dir/apps", } ], diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index c6ab40afc0..baa01224d1 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -57,7 +57,7 @@ def test_describe_streamlit(mock_connector, runner, mock_ctx): def _put_query(project_root: Path, source: str, dest: str): return dedent( - f"put file://{project_root.resolve() / 'output' / source} {dest} auto_compress=false parallel=4 overwrite=True" + f"put file://{project_root.resolve() / 'output' / 'deploy' / 'streamlit' / source} {dest} auto_compress=false parallel=4 overwrite=True" ) diff --git a/tests_integration/test_snowpark.py b/tests_integration/test_snowpark.py index c55f33ee90..025e887497 100644 --- a/tests_integration/test_snowpark.py +++ b/tests_integration/test_snowpark.py @@ -45,11 +45,14 @@ def test_snowpark_flow( ): database = test_database.upper() with project_directory("snowpark") as tmp_dir: + deploy_root = Path("output") / "deploy" / "snowpark" _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "my_snowpark_project", - Path("output") / "my_snowpark_project" / "app.zip", + Path("output") / "deploy", + deploy_root, + deploy_root / "my_snowpark_project", + deploy_root / "my_snowpark_project" / "app.zip", ] ) @@ -1245,12 +1248,15 @@ def test_snowpark_flow_v2( ): database = test_database.upper() with project_directory("snowpark_v2") as tmp_dir: + deploy_root = Path("output") / "deploy" / "snowpark" _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "app_1.zip", - Path("output") / "app_2.zip", - Path("output") / "c.py", + Path("output") / "deploy", + deploy_root, + deploy_root / "app_1.zip", + deploy_root / "app_2.zip", + deploy_root / "c.py", ] ) _test_steps.snowpark_deploy_should_finish_successfully_and_return( @@ -1392,12 +1398,15 @@ def test_snowpark_with_glob_patterns( ): database = test_database.upper() with project_directory("snowpark_glob_patterns"): + deploy_root = Path("output") / "deploy" / "snowpark" _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "app_1.zip", - Path("output") / "app_2.zip", - Path("output") / "e.py", + Path("output") / "deploy", + deploy_root, + deploy_root / "app_1.zip", + deploy_root / "app_2.zip", + deploy_root / "e.py", ] ) _test_steps.snowpark_deploy_should_finish_successfully_and_return( diff --git a/tests_integration/testing_utils/snowpark_utils.py b/tests_integration/testing_utils/snowpark_utils.py index 33412f2d0f..1a2a9d01dd 100644 --- a/tests_integration/testing_utils/snowpark_utils.py +++ b/tests_integration/testing_utils/snowpark_utils.py @@ -172,7 +172,9 @@ def snowpark_build_should_zip_files( if not no_dependencies: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): - additional_files.append(Path("output") / "dependencies.zip") + additional_files.append( + Path("output") / "deploy" / "snowpark" / "dependencies.zip" + ) else: additional_files.append(Path("dependencies.zip"))