From bf62cfb634f472c15daaf54708ec7b88703bf4af Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Wed, 11 Dec 2024 11:45:44 +0100 Subject: [PATCH] =?UTF-8?q?Moved=20Snowpark=20and=20Streamlit=20artifacts?= =?UTF-8?q?=20to=20separate=20directory=20in=20outpu=E2=80=A6=20(#1909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved Snowpark and Streamlit artifacts to separate directory in output/deploy --- .../cli/_plugins/snowpark/commands.py | 10 +- .../snowpark/snowpark_project_paths.py | 31 ++++-- .../cli/_plugins/streamlit/manager.py | 10 +- .../streamlit/streamlit_project_paths.py | 7 +- .../cli/api/project/project_paths.py | 16 ++- tests/snowpark/test_artifacts.py | 42 ++++---- tests/snowpark/test_build.py | 4 +- tests/snowpark/test_function.py | 25 ++++- tests/snowpark/test_procedure.py | 19 +++- tests/snowpark/test_project_paths.py | 79 ++++++++------ tests/streamlit/test_artifacts.py | 100 +++++++++++------- tests/streamlit/test_commands.py | 2 +- tests_integration/test_snowpark.py | 23 ++-- .../testing_utils/snowpark_utils.py | 4 +- 14 files changed, 242 insertions(+), 130 deletions(-) diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py index 28db01d068..37f093a930 100644 --- a/src/snowflake/cli/_plugins/snowpark/commands.py +++ b/src/snowflake/cli/_plugins/snowpark/commands.py @@ -220,19 +220,19 @@ def build_artifacts_mappings( ) -> Tuple[EntityToImportPathsMapping, StageToArtefactMapping]: stages_to_artifact_map: StageToArtefactMapping = defaultdict(set) entities_to_imports_map: EntityToImportPathsMapping = defaultdict(set) - for entity_id, entity in snowpark_entities.items(): + for name, entity in snowpark_entities.items(): stage = entity.stage required_artifacts = set() for artefact in entity.artifacts: artefact_dto = project_paths.get_artefact_dto(artefact) required_artifacts.add(artefact_dto) - entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage)) + entities_to_imports_map[name].add(artefact_dto.import_path(stage)) stages_to_artifact_map[stage].update(required_artifacts) deps_artefact = project_paths.get_dependencies_artefact() if deps_artefact.post_build_path.exists(): stages_to_artifact_map[stage].add(deps_artefact) - entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage)) + entities_to_imports_map[name].add(deps_artefact.import_path(stage)) return entities_to_imports_map, stages_to_artifact_map @@ -330,7 +330,7 @@ def build( anaconda_packages_manager = AnacondaPackagesManager() # Clean up deploy root - project_paths.remove_up_deploy_root() + project_paths.remove_up_bundle_root() # Resolve dependencies if project_paths.requirements.exists(): @@ -389,7 +389,7 @@ def build( for artefact in artifacts: bundle_map = BundleMap( project_root=artefact.project_root, - deploy_root=project_paths.deploy_root, + deploy_root=project_paths.bundle_root, ) bundle_map.add(PathMapping(src=str(artefact.path), dest=artefact.dest)) diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py index 6e103d840e..2bfe957cf2 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py @@ -25,7 +25,7 @@ from snowflake.cli.api.constants import DEPLOYMENT_STAGE from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.project_paths import ProjectPaths +from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.secure_path import SecurePath @@ -45,6 +45,7 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): return Artefact( project_root=self.project_root, + bundle_root=self.bundle_root, dest=artifact_path.dest, path=Path(artifact_path.src), ) @@ -57,7 +58,10 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: def get_dependencies_artefact(self) -> Artefact: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): return Artefact( - project_root=self.project_root, dest=None, path=Path("dependencies.zip") + project_root=self.project_root, + bundle_root=self.bundle_root, + dest=None, + path=Path("dependencies.zip"), ) else: return ArtefactOldBuild( @@ -74,19 +78,29 @@ def snowflake_requirements(self) -> SecurePath: def requirements(self) -> SecurePath: return SecurePath(self.path_relative_to_root(Path("requirements.txt"))) + @property + def bundle_root(self) -> Path: + return bundle_root(self.project_root, "snowpark") + @dataclass(unsafe_hash=True) class Artefact: """Helper for getting paths related to given artefact.""" project_root: Path + bundle_root: Path path: Path dest: str | None = None def __init__( - self, project_root: Path, path: Path, dest: Optional[str] = None + self, + project_root: Path, + bundle_root: Path, + path: Path, + dest: Optional[str] = None, ) -> None: self.project_root = project_root + self.bundle_root = bundle_root self.path = path self.dest = dest if self.dest and not self._is_dest_a_file() and not self.dest.endswith("/"): @@ -117,15 +131,15 @@ def post_build_path(self) -> Path: """ Returns post-build artefact path. Directories are mapped to corresponding .zip files. """ - deploy_root = self.deploy_root() + bundle_root = self.bundle_root path = ( self._path_until_asterisk() if glob.has_magic(str(self.path)) else self.path.parent ) if self._is_dest_a_file(): - return deploy_root / self.dest # type: ignore - return deploy_root / (self.dest or path) / self._artefact_name + return bundle_root / self.dest # type: ignore + return bundle_root / (self.dest or path) / self._artefact_name def upload_path(self, stage: FQN | str | None) -> str: """ @@ -153,9 +167,6 @@ def import_path(self, stage: FQN | str | None) -> str: """Path for UDF/sproc imports clause.""" return self.upload_path(stage) + self._artefact_name - def deploy_root(self) -> Path: - return self.project_root / "output" - def _is_dest_a_file(self) -> bool: if not self.dest: return False @@ -183,7 +194,7 @@ class ArtefactOldBuild(Artefact): dest: str | None = None def __init__(self, path: Path, dest: Optional[str] = None) -> None: - super().__init__(project_root=Path(), path=path, dest=dest) + super().__init__(project_root=Path(), bundle_root=Path(), path=path, dest=dest) @property def _artefact_name(self) -> str: diff --git a/src/snowflake/cli/_plugins/streamlit/manager.py b/src/snowflake/cli/_plugins/streamlit/manager.py index 4b677c903e..f6388a3f8f 100644 --- a/src/snowflake/cli/_plugins/streamlit/manager.py +++ b/src/snowflake/cli/_plugins/streamlit/manager.py @@ -68,29 +68,31 @@ def _put_streamlit_files( if not artifacts: return stage_manager = StageManager() + # We treat the bundle root as deploy root bundle_map = BundleMap( project_root=streamlit_project_paths.project_root, - deploy_root=streamlit_project_paths.deploy_root, + deploy_root=streamlit_project_paths.bundle_root, ) for artifact in artifacts: bundle_map.add(PathMapping(src=str(artifact.src), dest=artifact.dest)) # Clean up deploy root - streamlit_project_paths.remove_up_deploy_root() + streamlit_project_paths.remove_up_bundle_root() for (absolute_src, absolute_dest) in bundle_map.all_mappings( absolute=True, expand_directories=True ): if absolute_src.is_file(): + # We treat the bundle root as deploy root symlink_or_copy( absolute_src, absolute_dest, - deploy_root=streamlit_project_paths.deploy_root, + deploy_root=streamlit_project_paths.bundle_root, ) # Temporary solution, will be replaced with diff stage_path = ( PurePosixPath(absolute_dest) - .relative_to(streamlit_project_paths.deploy_root) + .relative_to(streamlit_project_paths.bundle_root) .parent ) full_stage_path = f"{stage_root}/{stage_path}".rstrip("/") diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py index fbe89959b2..df55dddc8e 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py @@ -14,8 +14,9 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path -from snowflake.cli.api.project.project_paths import ProjectPaths +from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root @dataclass @@ -23,3 +24,7 @@ class StreamlitProjectPaths(ProjectPaths): """ This class allows you to manage files paths related to given project. """ + + @property + def bundle_root(self) -> Path: + return bundle_root(self.project_root, "streamlit") diff --git a/src/snowflake/cli/api/project/project_paths.py b/src/snowflake/cli/api/project/project_paths.py index f6b2a14d99..544fa36dbf 100644 --- a/src/snowflake/cli/api/project/project_paths.py +++ b/src/snowflake/cli/api/project/project_paths.py @@ -8,9 +8,15 @@ class ProjectPaths: project_root: Path @property - def deploy_root(self) -> Path: - return self.project_root / "output" + def bundle_root(self) -> Path: + return bundle_root(self.project_root) - def remove_up_deploy_root(self) -> None: - if self.deploy_root.exists(): - rmtree(self.deploy_root) + def remove_up_bundle_root(self) -> None: + if self.bundle_root.exists(): + rmtree(self.bundle_root) + + +def bundle_root(root: Path, app_type: str | None = None) -> Path: + if app_type: + return root / "output" / "bundle" / app_type + return root / "output" / "bundle" diff --git a/tests/snowpark/test_artifacts.py b/tests/snowpark/test_artifacts.py index c573ab9b6d..fe16a06119 100644 --- a/tests/snowpark/test_artifacts.py +++ b/tests/snowpark/test_artifacts.py @@ -12,42 +12,44 @@ lambda _: True, ) +bundle_root = Path("output") / "bundle" / "snowpark" + @pytest.mark.parametrize( "artifacts, local_path, stage_path", [ - ("src", Path("output") / "src.zip", "/"), - ("src/", Path("output") / "src.zip", "/"), - ("src/*", Path("output") / "src.zip", "/"), - ("src/*.py", Path("output") / "src.zip", "/"), + ("src", bundle_root / "src.zip", "/"), + ("src/", bundle_root / "src.zip", "/"), + ("src/*", bundle_root / "src.zip", "/"), + ("src/*.py", bundle_root / "src.zip", "/"), ( "src/dir/dir_app.py", - Path("output") / "src" / "dir" / "dir_app.py", + bundle_root / "src" / "dir" / "dir_app.py", "/src/dir/", ), ( {"src": "src/**/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, - "output/source/dir/apps/dir_app.py", + bundle_root / "source" / "dir" / "apps" / "dir_app.py", "/source/dir/apps/", ), ], @@ -111,38 +113,38 @@ def test_build_and_deploy_with_artifacts( @pytest.mark.parametrize( "artifact, local_path, stage_path", [ - ("src", Path("output") / "src.zip", "/"), - ("src/", Path("output") / "src.zip", "/"), - ("src/*", Path("output") / "src.zip", "/"), - ("src/*.py", Path("output") / "src.zip", "/"), + ("src", bundle_root / "src.zip", "/"), + ("src/", bundle_root / "src.zip", "/"), + ("src/*", bundle_root / "src.zip", "/"), + ("src/*.py", bundle_root / "src.zip", "/"), ( "src/dir/dir_app.py", - Path("output") / "src" / "dir" / "dir_app.py", + bundle_root / "src" / "dir" / "dir_app.py", "/src/dir/", ), ( {"src": "src/**/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, - Path("output") / "source" / "dir" / "apps" / "dir_app.py", + bundle_root / "source" / "dir" / "apps" / "dir_app.py", "/source/dir/apps/", ), ], diff --git a/tests/snowpark/test_build.py b/tests/snowpark/test_build.py index 9298ed2b32..fb7c4841fc 100644 --- a/tests/snowpark/test_build.py +++ b/tests/snowpark/test_build.py @@ -58,7 +58,9 @@ def test_build_with_glob_patterns_in_artifacts( result = runner.invoke(["snowpark", "build", "--ignore-anaconda"]) assert result.exit_code == 0, result.output - _assert_zip_contains(tmp_dir / "output" / zip_name, expected_files) + _assert_zip_contains( + tmp_dir / "output" / "bundle" / "snowpark" / zip_name, expected_files + ) def _assert_zip_contains(app_zip: str, expected_files: Set[str]): diff --git a/tests/snowpark/test_function.py b/tests/snowpark/test_function.py index b071277602..741993c55a 100644 --- a/tests/snowpark/test_function.py +++ b/tests/snowpark/test_function.py @@ -236,7 +236,11 @@ def test_deploy_function_no_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), ] @@ -286,7 +290,11 @@ def test_deploy_function_needs_update_because_packages_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) @@ -348,8 +356,11 @@ def test_deploy_function_needs_update_because_handler_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" - f" auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) @@ -712,3 +723,9 @@ def test_command_aliases(mock_connector, runner, mock_ctx, command, parameters): queries = ctx.get_queries() assert queries[0] == queries[1] + + +def _put_query(project_root: Path, source: str, dest: str): + return dedent( + f"put file://{project_root.resolve() / 'output' / 'bundle' / 'snowpark' / source} {dest} auto_compress=false parallel=4 overwrite=True" + ) diff --git a/tests/snowpark/test_procedure.py b/tests/snowpark/test_procedure.py index 3b2069e7fd..e00152da3d 100644 --- a/tests/snowpark/test_procedure.py +++ b/tests/snowpark/test_procedure.py @@ -109,7 +109,11 @@ def test_deploy_procedure( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(tmp).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(tmp), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) @@ -199,8 +203,11 @@ def test_deploy_procedure_with_external_access( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" - f" auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) @@ -835,3 +842,9 @@ def test_snowpark_fail_if_no_active_warehouse(runner, mock_ctx, project_director "The command requires warehouse. No warehouse found in current connection." in result.output ) + + +def _put_query(project_root: Path, source: str, dest: str): + return dedent( + f"put file://{project_root.resolve() / 'output' / 'bundle' / 'snowpark' / source} {dest} auto_compress=false parallel=4 overwrite=True" + ) diff --git a/tests/snowpark/test_project_paths.py b/tests/snowpark/test_project_paths.py index 55796e1a98..19618424ac 100644 --- a/tests/snowpark/test_project_paths.py +++ b/tests/snowpark/test_project_paths.py @@ -4,6 +4,9 @@ import pytest from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact +bundle_root = Path("output") / "bundle" / "snowpark" +absolute_bundle_root = Path.cwd().absolute() / "output" / "bundle" / "snowpark" + @pytest.mark.parametrize( "path, dest, is_file, expected_path", @@ -31,7 +34,7 @@ def test_artifact_import_path(mock_ctx_context, path, dest, is_file, expected_pa stage = "stage" with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - import_path = Artefact(Path(), Path(path), dest).import_path(stage) + import_path = Artefact(Path(), bundle_root, Path(path), dest).import_path(stage) assert import_path == expected_path @@ -61,7 +64,9 @@ def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_pa mock_ctx_context.return_value.connection = mock_connection with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - upload_path = Artefact(Path(), Path(path), dest).upload_path("stage") + upload_path = Artefact(Path(), bundle_root, Path(path), dest).upload_path( + "stage" + ) assert upload_path == expected_path @@ -69,39 +74,41 @@ def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_pa @pytest.mark.parametrize( "path, dest, is_file, expected_path", [ - ("src", None, False, Path("output") / "src.zip"), - ("src/", None, False, Path("output") / "src.zip"), - ("src", "source", False, Path("output") / "source" / "src.zip"), - ("src/app.py", None, True, Path("output") / "src" / "app.py"), + ("src", None, False, bundle_root / "src.zip"), + ("src/", None, False, bundle_root / "src.zip"), + ("src", "source", False, bundle_root / "source" / "src.zip"), + ("src/app.py", None, True, bundle_root / "src" / "app.py"), ( "src/app.py", "source/new_app.py", True, - Path("output") / "source" / "new_app.py", + bundle_root / "source" / "new_app.py", ), - ("src/*", "source/new_app.py", True, Path("output") / "source" / "new_app.py"), + ("src/*", "source/new_app.py", True, bundle_root / "source" / "new_app.py"), ( "src/dir/dir2/app.py", None, True, - Path("output") / "src" / "dir" / "dir2" / "app.py", + bundle_root / "src" / "dir" / "dir2" / "app.py", ), ( "src/dir/dir2/app.py", "source/", True, - Path("output") / "source" / "app.py", + bundle_root / "source" / "app.py", ), - ("src/*", "source/", False, Path("output") / "source" / "src.zip"), - ("src/**/*.py", None, False, Path("output") / "src.zip"), - ("src/**/*.py", "source/", False, Path("output") / "source" / "src.zip"), - ("src/app*", None, False, Path("output") / "src.zip"), - ("src/app[1-5].py", None, False, Path("output") / "src.zip"), + ("src/*", "source/", False, bundle_root / "source" / "src.zip"), + ("src/**/*.py", None, False, bundle_root / "src.zip"), + ("src/**/*.py", "source/", False, bundle_root / "source" / "src.zip"), + ("src/app*", None, False, bundle_root / "src.zip"), + ("src/app[1-5].py", None, False, bundle_root / "src.zip"), ], ) def test_artifact_post_build_path(path, dest, is_file, expected_path): with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - post_build_path = Artefact(Path(), Path(path), dest).post_build_path + post_build_path = Artefact( + Path(), bundle_root, Path(path), dest + ).post_build_path assert post_build_path == expected_path @@ -134,7 +141,12 @@ def test_artifact_import_path_from_other_directory( stage = "stage" with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - import_path = Artefact(Path("/tmp"), Path(path), dest).import_path(stage) + import_path = Artefact( + Path("/tmp"), + Path("/tmp") / "output" / "deploy" / "snowpark", + Path(path), + dest, + ).import_path(stage) assert import_path == expected_path @@ -166,7 +178,9 @@ def test_artifact_upload_path_from_other_directory( mock_ctx_context.return_value.connection = mock_connection with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - upload_path = Artefact(Path("/tmp"), Path(path), dest).upload_path("stage") + upload_path = Artefact( + Path("/tmp"), Path("/tmp") / "output" / "deploy", Path(path), dest + ).upload_path("stage") assert upload_path == expected_path @@ -174,48 +188,48 @@ def test_artifact_upload_path_from_other_directory( @pytest.mark.parametrize( "path, dest, is_file, expected_path", [ - ("src", None, False, Path.cwd().absolute() / "output" / "src.zip"), - ("src/", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src", None, False, absolute_bundle_root / "src.zip"), + ("src/", None, False, absolute_bundle_root / "src.zip"), ( "src", "source", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_bundle_root / "source" / "src.zip", ), - ("src/app.py", None, True, Path.cwd().absolute() / "output" / "src" / "app.py"), + ("src/app.py", None, True, absolute_bundle_root / "src" / "app.py"), ( "src/app.py", "source/new_app.py", True, - Path.cwd().absolute() / "output" / "source" / "new_app.py", + absolute_bundle_root / "source" / "new_app.py", ), ( "src/dir/dir2/app.py", None, True, - Path.cwd().absolute() / "output" / "src" / "dir" / "dir2" / "app.py", + absolute_bundle_root / "src" / "dir" / "dir2" / "app.py", ), ( "src/dir/dir2/app.py", "source/", True, - Path.cwd().absolute() / "output" / "source" / "app.py", + absolute_bundle_root / "source" / "app.py", ), ( "src/*", "source/", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_bundle_root / "source" / "src.zip", ), - ("src/**/*.py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/**/*.py", None, False, absolute_bundle_root / "src.zip"), ( "src/**/*.py", "source/", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_bundle_root / "source" / "src.zip", ), - ("src/app*", None, False, Path.cwd().absolute() / "output" / "src.zip"), - ("src/app[1-5].py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/app*", None, False, absolute_bundle_root / "src.zip"), + ("src/app[1-5].py", None, False, absolute_bundle_root / "src.zip"), ], ) def test_artifact_post_build_path_from_other_directory( @@ -223,7 +237,10 @@ def test_artifact_post_build_path_from_other_directory( ): with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): post_build_path = Artefact( - Path.cwd().absolute(), Path(path), dest + Path.cwd().absolute(), + absolute_bundle_root, + Path(path), + dest, ).post_build_path assert post_build_path == expected_path diff --git a/tests/streamlit/test_artifacts.py b/tests/streamlit/test_artifacts.py index d63f3033df..fe05ffab9a 100644 --- a/tests/streamlit/test_artifacts.py +++ b/tests/streamlit/test_artifacts.py @@ -6,6 +6,8 @@ from snowflake.cli._plugins.connection.util import UIParameter from snowflake.connector.compat import IS_WINDOWS +bundle_root = Path("output") / "bundle" / "streamlit" + @pytest.mark.parametrize( "artifacts, paths", @@ -13,9 +15,9 @@ ( "src", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], @@ -23,9 +25,9 @@ ( "src/", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], @@ -33,19 +35,19 @@ ( "src/*", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], ), - ("src/*.py", [{"local": Path("output") / "src" / "app.py", "stage": "/src"}]), + ("src/*.py", [{"local": bundle_root / "src" / "app.py", "stage": "/src"}]), ( "src/dir/dir_app.py", [ { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", } ], @@ -53,10 +55,10 @@ ( {"src": "src/**/*", "dest": "source/"}, [ - {"local": Path("output") / "source" / "app.py", "stage": "/source"}, - {"local": Path("output") / "source" / "dir_app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "dir_app.py", "stage": "/source"}, { - "local": Path("output") / "source" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "dir" / "dir_app.py", "stage": "/source/dir", }, ], @@ -65,11 +67,11 @@ {"src": "src", "dest": "source/"}, [ { - "local": Path("output") / "source" / "src" / "app.py", + "local": bundle_root / "source" / "src" / "app.py", "stage": "/source/src", }, { - "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -78,11 +80,11 @@ {"src": "src/", "dest": "source/"}, [ { - "local": Path("output") / "source" / "src" / "app.py", + "local": bundle_root / "source" / "src" / "app.py", "stage": "/source/src", }, { - "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -90,9 +92,9 @@ ( {"src": "src/*", "dest": "source/"}, [ - {"local": Path("output") / "source" / "app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, { - "local": Path("output") / "source" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "dir" / "dir_app.py", "stage": "/source/dir", }, ], @@ -101,7 +103,7 @@ {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, [ { - "local": Path("output") / "source" / "dir" / "apps" / "dir_app.py", + "local": bundle_root / "source" / "dir" / "apps" / "dir_app.py", "stage": "/source/dir/apps", } ], @@ -179,43 +181,63 @@ def test_deploy_with_artifacts( ( "src", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), ( "src/", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), ( "src/*", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), - ("src/*.py", [{"local": "output/src/app.py", "stage": "/src"}]), + ("src/*.py", [{"local": bundle_root / "src" / "app.py", "stage": "/src"}]), ( "src/dir/dir_app.py", - [{"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}], + [ + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + } + ], ), ( {"src": "src/**/*", "dest": "source/"}, [ - {"local": "output/source/app.py", "stage": "/source"}, - {"local": "output/source/dir_app.py", "stage": "/source"}, - {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "dir_app.py", "stage": "/source"}, + { + "local": bundle_root / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, ], ), ( {"src": "src", "dest": "source/"}, [ - {"local": "output/source/src/app.py", "stage": "/source/src"}, { - "local": "output/source/src/dir/dir_app.py", + "local": bundle_root / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -223,9 +245,12 @@ def test_deploy_with_artifacts( ( {"src": "src/", "dest": "source/"}, [ - {"local": "output/source/src/app.py", "stage": "/source/src"}, { - "local": "output/source/src/dir/dir_app.py", + "local": bundle_root / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -233,15 +258,18 @@ def test_deploy_with_artifacts( ( {"src": "src/*", "dest": "source/"}, [ - {"local": "output/source/app.py", "stage": "/source"}, - {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, + { + "local": bundle_root / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, ], ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, [ { - "local": "output/source/dir/apps/dir_app.py", + "local": bundle_root / "source" / "dir" / "apps" / "dir_app.py", "stage": "/source/dir/apps", } ], diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index 9a859ad030..8eff0b4153 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -57,7 +57,7 @@ def test_describe_streamlit(mock_connector, runner, mock_ctx): def _put_query(project_root: Path, source: str, dest: str): return dedent( - f"put file://{project_root.resolve() / 'output' / source} {dest} auto_compress=false parallel=4 overwrite=True" + f"put file://{project_root.resolve() / 'output' / 'bundle' / 'streamlit' / source} {dest} auto_compress=false parallel=4 overwrite=True" ) diff --git a/tests_integration/test_snowpark.py b/tests_integration/test_snowpark.py index c55f33ee90..49cd4d5c96 100644 --- a/tests_integration/test_snowpark.py +++ b/tests_integration/test_snowpark.py @@ -33,6 +33,7 @@ STAGE_NAME = "dev_deployment" RETURN_TYPE = "VARCHAR" if IS_QA else "VARCHAR(16777216)" +bundle_root = Path("output") / "bundle" / "snowpark" @pytest.mark.integration @@ -48,8 +49,10 @@ def test_snowpark_flow( _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "my_snowpark_project", - Path("output") / "my_snowpark_project" / "app.zip", + Path("output") / "bundle", + bundle_root, + bundle_root / "my_snowpark_project", + bundle_root / "my_snowpark_project" / "app.zip", ] ) @@ -1248,9 +1251,11 @@ def test_snowpark_flow_v2( _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "app_1.zip", - Path("output") / "app_2.zip", - Path("output") / "c.py", + Path("output") / "bundle", + bundle_root, + bundle_root / "app_1.zip", + bundle_root / "app_2.zip", + bundle_root / "c.py", ] ) _test_steps.snowpark_deploy_should_finish_successfully_and_return( @@ -1395,9 +1400,11 @@ def test_snowpark_with_glob_patterns( _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "app_1.zip", - Path("output") / "app_2.zip", - Path("output") / "e.py", + Path("output") / "bundle", + bundle_root, + bundle_root / "app_1.zip", + bundle_root / "app_2.zip", + bundle_root / "e.py", ] ) _test_steps.snowpark_deploy_should_finish_successfully_and_return( diff --git a/tests_integration/testing_utils/snowpark_utils.py b/tests_integration/testing_utils/snowpark_utils.py index 33412f2d0f..3cb4510bc7 100644 --- a/tests_integration/testing_utils/snowpark_utils.py +++ b/tests_integration/testing_utils/snowpark_utils.py @@ -172,7 +172,9 @@ def snowpark_build_should_zip_files( if not no_dependencies: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): - additional_files.append(Path("output") / "dependencies.zip") + additional_files.append( + Path("output") / "bundle" / "snowpark" / "dependencies.zip" + ) else: additional_files.append(Path("dependencies.zip"))