From ebb56f8bd95556b0c998e25a6f97ccf1ac3fa116 Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Mon, 18 Nov 2024 12:58:42 +0100 Subject: [PATCH 01/10] Added glob support for snowpark and streamlit (#1862) * Added glob support for Snowpark and Streamlit * Added glob support for Snowpark and Streamlit * Added glob support for Snowpark and Streamlit * Changed PathMapping.src to Path * Changes after review * Changes after review 2 * Revert "Changed PathMapping.src to Path" This reverts commit 3eb543df96a6a3eb005d5beb20712e01208ce44c. * Changes after review 3 * Changes after review 4 * Added full global support * Renamed feature flag and added release notes * Removed or None --- RELEASE-NOTES.md | 2 + .../nativeapp/entities/application_package.py | 9 +- .../cli/_plugins/snowpark/commands.py | 61 +- .../snowpark/snowpark_entity_model.py | 44 +- .../snowpark/snowpark_project_paths.py | 141 ++++- src/snowflake/cli/_plugins/snowpark/zipper.py | 29 + .../cli/_plugins/streamlit/commands.py | 11 +- .../cli/_plugins/streamlit/manager.py | 72 ++- .../streamlit/streamlit_entity_model.py | 35 +- .../streamlit/streamlit_project_paths.py | 25 + src/snowflake/cli/api/feature_flags.py | 1 + .../cli/api/project/definition_conversion.py | 5 +- .../cli/api/project/project_paths.py | 16 + .../cli/api/project/schemas/commons.py | 5 + .../schemas/v1/native_app/native_app.py | 11 +- .../helpers/__snapshots__/test_v1_to_v2.ambr | 18 +- tests/project/test_project_definition_v2.py | 2 +- .../test_function_old_build.ambr | 62 ++ tests/snowpark/__snapshots__/test_models.ambr | 14 + .../test_procedure_old_build.ambr | 56 ++ tests/snowpark/test_artifacts.py | 215 +++++++ tests/snowpark/test_build.py | 38 +- tests/snowpark/test_function.py | 108 +++- tests/snowpark/test_function_old_build.py | 506 ++++++++++++++++ tests/snowpark/test_models.py | 14 + tests/snowpark/test_procedure.py | 178 +++++- tests/snowpark/test_procedure_old_build.py | 555 ++++++++++++++++++ tests/snowpark/test_project_paths.py | 229 ++++++++ tests/streamlit/test_artifacts.py | 316 ++++++++++ tests/streamlit/test_commands.py | 164 +++--- tests/streamlit/test_streamlit_manager.py | 27 +- .../projects/glob_patterns/environment.yml | 5 + .../test_data/projects/glob_patterns/main.py | 18 + .../projects/glob_patterns/pages/my_page.py | 0 .../projects/glob_patterns/snowflake.yml | 23 + .../projects/glob_patterns/src/app.py | 12 + .../projects/glob_patterns/src/dir/dir_app.py | 2 + .../projects/glob_patterns/streamlit_app.py | 0 .../glob_patterns_zip/commons/helpers.py | 0 .../glob_patterns_zip/environment.yml | 5 + .../projects/glob_patterns_zip/main.py | 18 + .../projects/glob_patterns_zip/snowflake.yml | 23 + .../projects/glob_patterns_zip/src/app.py | 12 + .../glob_patterns_zip/src/dir/dir_app.py | 2 + .../glob_patterns_zip/streamlit_app.py | 0 tests/testing_utils/fixtures.py | 9 + tests_integration/conftest.py | 10 + .../snowpark_glob_patterns/app_1/a.py | 12 + .../snowpark_glob_patterns/app_1/b.py | 10 + .../snowpark_glob_patterns/app_2/c.py | 12 + .../snowpark_glob_patterns/app_2/d.py | 10 + .../projects/snowpark_glob_patterns/e.py | 9 + .../snowpark_glob_patterns/requirements.txt | 1 + .../snowpark_glob_patterns/snowflake.yml | 43 ++ tests_integration/test_snowpark.py | 429 ++++++++++++++ .../testing_utils/snowpark_utils.py | 6 +- 56 files changed, 3415 insertions(+), 225 deletions(-) create mode 100644 src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py create mode 100644 src/snowflake/cli/api/project/project_paths.py create mode 100644 src/snowflake/cli/api/project/schemas/commons.py create mode 100644 tests/snowpark/__snapshots__/test_function_old_build.ambr create mode 100644 tests/snowpark/__snapshots__/test_models.ambr create mode 100644 tests/snowpark/__snapshots__/test_procedure_old_build.ambr create mode 100644 tests/snowpark/test_artifacts.py create mode 100644 tests/snowpark/test_function_old_build.py create mode 100644 tests/snowpark/test_procedure_old_build.py create mode 100644 tests/snowpark/test_project_paths.py create mode 100644 tests/streamlit/test_artifacts.py create mode 100644 tests/test_data/projects/glob_patterns/environment.yml create mode 100644 tests/test_data/projects/glob_patterns/main.py create mode 100644 tests/test_data/projects/glob_patterns/pages/my_page.py create mode 100644 tests/test_data/projects/glob_patterns/snowflake.yml create mode 100644 tests/test_data/projects/glob_patterns/src/app.py create mode 100644 tests/test_data/projects/glob_patterns/src/dir/dir_app.py create mode 100644 tests/test_data/projects/glob_patterns/streamlit_app.py create mode 100644 tests/test_data/projects/glob_patterns_zip/commons/helpers.py create mode 100644 tests/test_data/projects/glob_patterns_zip/environment.yml create mode 100644 tests/test_data/projects/glob_patterns_zip/main.py create mode 100644 tests/test_data/projects/glob_patterns_zip/snowflake.yml create mode 100644 tests/test_data/projects/glob_patterns_zip/src/app.py create mode 100644 tests/test_data/projects/glob_patterns_zip/src/dir/dir_app.py create mode 100644 tests/test_data/projects/glob_patterns_zip/streamlit_app.py create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/app_1/a.py create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/app_1/b.py create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/app_2/c.py create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/app_2/d.py create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/e.py create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/requirements.txt create mode 100644 tests_integration/test_data/projects/snowpark_glob_patterns/snowflake.yml diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 1f002a8c12..9c4e751adf 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -51,6 +51,8 @@ * `--follow`: Stream logs in real-time. * `--follow-interval`: Set custom polling intervals during log streaming. * `snow connection add` supports `--no-interactive` flag to skip interactive prompts. +* Added support for glob pattern in artifact paths in snowflake.yml for Streamlit. +* Added support for glob pattern in artifact paths in snowflake.yml for Snowpark, requires ENABLE_SNOWPARK_GLOB_SUPPORT feature flag. ## Fixes and improvements * `snow --info` callback returns information about `SNOWFLAKE_HOME` variable. diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 8f76c7ac4c..fc5a675fb6 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -79,6 +79,7 @@ ) from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError +from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.entities.common import ( EntityModelBase, Identifier, @@ -147,7 +148,7 @@ class ApplicationPackageChildField(UpdatableModel): class ApplicationPackageEntityModel(EntityModelBase): type: Literal["application package"] = DiscriminatorField() # noqa: A003 - artifacts: List[Union[PathMapping, str]] = Field( + artifacts: Artifacts = Field( title="List of paths or file source/destination pairs to add to the deploy root", ) bundle_root: Optional[str] = Field( @@ -211,10 +212,8 @@ def append_test_resource_suffix_to_identifier( @field_validator("artifacts") @classmethod - def transform_artifacts( - cls, orig_artifacts: List[Union[PathMapping, str]] - ) -> List[PathMapping]: - transformed_artifacts = [] + def transform_artifacts(cls, orig_artifacts: Artifacts) -> List[PathMapping]: + transformed_artifacts: List[PathMapping] = [] if orig_artifacts is None: return transformed_artifacts diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py index ba73ca59e4..cbe9cea568 100644 --- a/src/snowflake/cli/_plugins/snowpark/commands.py +++ b/src/snowflake/cli/_plugins/snowpark/commands.py @@ -20,6 +20,7 @@ import typer from click import ClickException, UsageError +from snowflake.cli._plugins.nativeapp.artifacts import BundleMap, symlink_or_copy from snowflake.cli._plugins.object.commands import ( describe as object_describe, ) @@ -59,7 +60,7 @@ IndexUrlOption, SkipVersionCheckOption, ) -from snowflake.cli._plugins.snowpark.zipper import zip_dir +from snowflake.cli._plugins.snowpark.zipper import zip_dir, zip_dir_using_bundle_map from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.cli_global_context import ( get_cli_context, @@ -81,6 +82,7 @@ from snowflake.cli.api.exceptions import ( SecretsWithoutExternalAccessIntegrationError, ) +from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.output.types import ( CollectionResult, @@ -95,6 +97,7 @@ ProjectDefinition, ProjectDefinitionV2, ) +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.secure_path import SecurePath from snowflake.connector import DictCursor, ProgrammingError from snowflake.connector.cursor import SnowflakeCursor @@ -225,8 +228,8 @@ def build_artifacts_mappings( entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage)) stages_to_artifact_map[stage].update(required_artifacts) - if project_paths.dependencies.exists(): - deps_artefact = project_paths.get_dependencies_artefact() + deps_artefact = project_paths.get_dependencies_artefact() + if deps_artefact.post_build_path.exists(): stages_to_artifact_map[stage].add(deps_artefact) entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage)) return entities_to_imports_map, stages_to_artifact_map @@ -239,11 +242,12 @@ def create_stages_and_upload_artifacts(stages_to_artifact_map: StageToArtefactMa stage = FQN.from_stage(stage).using_context() stage_manager.create(fqn=stage, comment="deployments managed by Snowflake CLI") for artefact in artifacts: + post_build_path = artefact.post_build_path cli_console.step( - f"Uploading {artefact.post_build_path.name} to {artefact.upload_path(stage)}" + f"Uploading {post_build_path.name} to {artefact.upload_path(stage)}" ) stage_manager.put( - local_path=artefact.post_build_path, + local_path=post_build_path, stage_path=artefact.upload_path(stage), overwrite=True, ) @@ -324,6 +328,9 @@ def build( anaconda_packages_manager = AnacondaPackagesManager() + # Clean up deploy root + project_paths.remove_up_deploy_root() + # Resolve dependencies if project_paths.requirements.exists(): with ( @@ -362,22 +369,50 @@ def build( ) if any(temp_deps_dir.path.iterdir()): - cli_console.step(f"Creating {project_paths.dependencies.name}") + dep_artifact = project_paths.get_dependencies_artefact() + cli_console.step(f"Creating {dep_artifact.path.name}") zip_dir( source=temp_deps_dir.path, - dest_zip=project_paths.dependencies, + dest_zip=dep_artifact.post_build_path, ) else: cli_console.step(f"No external dependencies.") artifacts = set() - for entity in get_snowpark_entities(pd).values(): - artifacts.update(entity.artifacts) - with cli_console.phase("Preparing artifacts for source code"): - for artefact in artifacts: - artefact_dto = project_paths.get_artefact_dto(artefact) - artefact_dto.build() + if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): + for entity in get_snowpark_entities(pd).values(): + for artifact in entity.artifacts: + artifacts.add(project_paths.get_artefact_dto(artifact)) + + for artefact in artifacts: + bundle_map = BundleMap( + project_root=artefact.project_root, + deploy_root=project_paths.deploy_root, + ) + bundle_map.add(PathMapping(src=str(artefact.path), dest=artefact.dest)) + + if artefact.path.is_file(): + for (absolute_src, absolute_dest) in bundle_map.all_mappings( + absolute=True, expand_directories=False + ): + symlink_or_copy( + absolute_src, + absolute_dest, + deploy_root=bundle_map.deploy_root(), + ) + else: + zip_dir_using_bundle_map( + bundle_map=bundle_map, + dest_zip=artefact.post_build_path, + ) + else: + for entity in get_snowpark_entities(pd).values(): + for artifact in entity.artifacts: + artifacts.add(project_paths.get_artefact_dto(artifact)) + + for artefact in artifacts: + artefact.build() return MessageResult(f"Build done.") diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py b/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py index a92716280c..5b355b804a 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py @@ -14,11 +14,13 @@ from __future__ import annotations -from pathlib import Path +import glob from typing import List, Literal, Optional, Union from pydantic import Field, field_validator +from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN +from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.entities.common import ( EntityModelBase, ExternalAccessBaseModel, @@ -26,24 +28,11 @@ ) from snowflake.cli.api.project.schemas.updatable_model import ( DiscriminatorField, - UpdatableModel, ) +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.project.schemas.v1.snowpark.argument import Argument -class PathMapping(UpdatableModel): - class Config: - frozen = True - - src: Path = Field(title="Source path (relative to project root)", default=None) - - dest: Optional[str] = Field( - title="Destination path on stage", - description="Paths are relative to stage root; paths ending with a slash indicate that the destination is a directory which source files should be copied into.", - default=None, - ) - - class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel): handler: str = Field( title="Function’s or procedure’s implementation of the object inside source module", @@ -59,17 +48,24 @@ class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseM title="Python version to use when executing ", default=None ) stage: str = Field(title="Stage in which artifacts will be stored") - artifacts: List[Union[PathMapping, str]] = Field(title="List of required sources") + artifacts: Artifacts = Field(title="List of required sources") @field_validator("artifacts") @classmethod def _convert_artifacts(cls, artifacts: Union[dict, str]): _artifacts = [] - for artefact in artifacts: - if isinstance(artefact, PathMapping): - _artifacts.append(artefact) + for artifact in artifacts: + if ( + (isinstance(artifact, str) and glob.has_magic(artifact)) + or (isinstance(artifact, PathMapping) and glob.has_magic(artifact.src)) + ) and FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_disabled(): + raise ValueError( + "If you want to use glob patterns in artifacts, you need to enable the Snowpark new build feature flag (ENABLE_SNOWPARK_GLOB_SUPPORT=true)" + ) + if isinstance(artifact, PathMapping): + _artifacts.append(artifact) else: - _artifacts.append(PathMapping(src=artefact)) + _artifacts.append(PathMapping(src=artifact)) return _artifacts @field_validator("runtime") @@ -79,14 +75,6 @@ def convert_runtime(cls, runtime_input: Union[str, float]) -> str: return str(runtime_input) return runtime_input - @field_validator("artifacts") - @classmethod - def validate_artifacts(cls, artifacts: List[Path]) -> List[Path]: - for artefact in artifacts: - if "*" in str(artefact): - raise ValueError("Glob patterns not supported for Snowpark artifacts.") - return artifacts - @property def udf_sproc_identifier(self) -> UdfSprocIdentifier: return UdfSprocIdentifier.from_definition(self) diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py index 7e155a9344..cbc05e2c2d 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py @@ -13,38 +13,56 @@ # limitations under the License. from __future__ import annotations +import glob +import os +import re from dataclasses import dataclass from pathlib import Path, PurePosixPath +from typing import Optional from snowflake.cli._plugins.snowpark.snowpark_entity_model import PathMapping from snowflake.cli._plugins.snowpark.zipper import zip_dir from snowflake.cli.api.console import cli_console from snowflake.cli.api.constants import DEPLOYMENT_STAGE +from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN +from snowflake.cli.api.project.project_paths import ProjectPaths from snowflake.cli.api.secure_path import SecurePath @dataclass -class SnowparkProjectPaths: +class SnowparkProjectPaths(ProjectPaths): """ - This class represents allows you to manage files paths related to given project. + This class allows you to manage files paths related to given project. """ - project_root: Path - def path_relative_to_root(self, artifact_path: Path) -> Path: if artifact_path.is_absolute(): return artifact_path return (self.project_root / artifact_path).resolve() def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: - return Artefact( - dest=artifact_path.dest, - path=self.path_relative_to_root(artifact_path.src), - ) + if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): + return Artefact( + project_root=self.project_root, + dest=artifact_path.dest, + path=Path(artifact_path.src), + ) + else: + return ArtefactOldBuild( + dest=artifact_path.dest, + path=self.path_relative_to_root(Path(artifact_path.src)), + ) def get_dependencies_artefact(self) -> Artefact: - return Artefact(dest=None, path=self.dependencies) + if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): + return Artefact( + project_root=self.project_root, dest=None, path=Path("dependencies.zip") + ) + else: + return ArtefactOldBuild( + dest=None, path=self.path_relative_to_root(Path("dependencies.zip")) + ) @property def snowflake_requirements(self) -> SecurePath: @@ -56,18 +74,117 @@ def snowflake_requirements(self) -> SecurePath: def requirements(self) -> SecurePath: return SecurePath(self.path_relative_to_root(Path("requirements.txt"))) + +@dataclass(unsafe_hash=True) +class Artefact: + """Helper for getting paths related to given artefact.""" + + project_root: Path + path: Path + dest: str | None = None + + def __init__( + self, project_root: Path, path: Path, dest: Optional[str] = None + ) -> None: + self.project_root = project_root + self.path = path + self.dest = dest + if self.dest and not self._is_dest_a_file() and not self.dest.endswith("/"): + self.dest = self.dest + "/" + @property - def dependencies(self) -> Path: - return self.path_relative_to_root(Path("dependencies.zip")) + def _artefact_name(self) -> str: + if glob.has_magic(str(self.path)): + last_part = None + for part in self.path.parts: + if glob.has_magic(part): + break + else: + last_part = part + if not last_part: + last_part = os.path.commonpath( + [str(self.path), str(self.path.absolute())] + ) + return last_part + ".zip" + if (self.project_root / self.path).is_dir(): + return self.path.stem + ".zip" + if (self.project_root / self.path).is_file() and self._is_dest_a_file(): + return Path(self.dest).name # type: ignore + return self.path.name + + @property + def post_build_path(self) -> Path: + """ + Returns post-build artefact path. Directories are mapped to corresponding .zip files. + """ + deploy_root = self.deploy_root() + path = ( + self._path_until_asterisk() + if glob.has_magic(str(self.path)) + else self.path.parent + ) + if self._is_dest_a_file(): + return deploy_root / self.dest # type: ignore + return deploy_root / (self.dest or path) / self._artefact_name + + def upload_path(self, stage: FQN | str | None) -> str: + """ + Path on stage to which the artefact should be uploaded. + """ + stage = stage or DEPLOYMENT_STAGE + if isinstance(stage, str): + stage = FQN.from_stage(stage).using_context() + + stage_path = PurePosixPath(f"@{stage}") + if self.dest: + stage_path /= ( + PurePosixPath(self.dest).parent if self._is_dest_a_file() else self.dest + ) + else: + stage_path /= ( + self._path_until_asterisk() + if glob.has_magic(str(self.path)) + else PurePosixPath(self.path).parent + ) + + return str(stage_path) + "/" + + def import_path(self, stage: FQN | str | None) -> str: + """Path for UDF/sproc imports clause.""" + return self.upload_path(stage) + self._artefact_name + + def deploy_root(self) -> Path: + return self.project_root / "output" + + def _is_dest_a_file(self) -> bool: + if not self.dest: + return False + return re.search(r"\.[a-zA-Z0-9]{2,4}$", self.dest) is not None + + def _path_until_asterisk(self) -> Path: + path = [] + for part in self.path.parts: + if glob.has_magic(part): + break + else: + path.append(part) + return Path(*path[:-1]) + + # Can be removed after removing ENABLE_SNOWPARK_GLOB_SUPPORT feature flag. + def build(self) -> None: + raise NotImplementedError("Not implemented in Artefact class.") @dataclass(unsafe_hash=True) -class Artefact: +class ArtefactOldBuild(Artefact): """Helper for getting paths related to given artefact.""" path: Path dest: str | None = None + def __init__(self, path: Path, dest: Optional[str] = None) -> None: + super().__init__(project_root=Path(), path=path, dest=dest) + @property def _artefact_name(self) -> str: if self.path.is_dir(): diff --git a/src/snowflake/cli/_plugins/snowpark/zipper.py b/src/snowflake/cli/_plugins/snowpark/zipper.py index abcb457220..34889680d0 100644 --- a/src/snowflake/cli/_plugins/snowpark/zipper.py +++ b/src/snowflake/cli/_plugins/snowpark/zipper.py @@ -20,6 +20,9 @@ from typing import Dict, List, Literal from zipfile import ZIP_DEFLATED, ZipFile +from snowflake.cli._plugins.nativeapp.artifacts import BundleMap +from snowflake.cli.api.console import cli_console + log = logging.getLogger(__name__) IGNORED_FILES = [ @@ -64,6 +67,9 @@ def zip_dir( mode: Literal["r", "w", "x", "a"] = "w", ) -> None: + if not dest_zip.parent.exists(): + dest_zip.parent.mkdir(parents=True) + if isinstance(source, Path): source = [source] @@ -79,6 +85,29 @@ def zip_dir( package_zip.write(file, arcname=file.relative_to(src)) +def zip_dir_using_bundle_map( + bundle_map: BundleMap, + dest_zip: Path, + mode: Literal["r", "w", "x", "a"] = "w", +) -> None: + if not dest_zip.parent.exists(): + dest_zip.parent.mkdir(parents=True) + + with ZipFile(dest_zip, mode, ZIP_DEFLATED, allowZip64=True) as package_zip: + cli_console.step(f"Creating: {dest_zip}") + for src, _ in bundle_map.all_mappings(expand_directories=True): + if src.is_file(): + log.debug("Adding %s to %s", src, dest_zip) + package_zip.write(src, arcname=_path_without_top_level_directory(src)) + + +def _path_without_top_level_directory(path: Path) -> str: + path_parts = path.parts + if len(path_parts) > 1: + return str(Path(*path_parts[1:])) + return str(path) + + def _to_be_zipped(file: Path) -> bool: for pattern in IGNORED_FILES: # This has to be a string because of fnmatch diff --git a/src/snowflake/cli/_plugins/streamlit/commands.py b/src/snowflake/cli/_plugins/streamlit/commands.py index 9449cbc66a..5602dd8e9a 100644 --- a/src/snowflake/cli/_plugins/streamlit/commands.py +++ b/src/snowflake/cli/_plugins/streamlit/commands.py @@ -29,6 +29,9 @@ from snowflake.cli._plugins.streamlit.streamlit_entity_model import ( StreamlitEntityModel, ) +from snowflake.cli._plugins.streamlit.streamlit_project_paths import ( + StreamlitProjectPaths, +) from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.commands.decorators import ( with_experimental_behaviour, @@ -156,6 +159,8 @@ def streamlit_deploy( entity_type="streamlit" ) + streamlit_project_paths = StreamlitProjectPaths(cli_context.project_root) + if not streamlits: raise NoProjectDefinitionError( project_type="streamlit", project_root=cli_context.project_root @@ -174,7 +179,11 @@ def streamlit_deploy( # Get first streamlit streamlit: StreamlitEntityModel = streamlits[entity_id] - url = StreamlitManager().deploy(streamlit=streamlit, replace=replace) + url = StreamlitManager().deploy( + streamlit=streamlit, + streamlit_project_paths=streamlit_project_paths, + replace=replace, + ) if open_: typer.launch(url) diff --git a/src/snowflake/cli/_plugins/streamlit/manager.py b/src/snowflake/cli/_plugins/streamlit/manager.py index 3eabd528d9..1ec889089a 100644 --- a/src/snowflake/cli/_plugins/streamlit/manager.py +++ b/src/snowflake/cli/_plugins/streamlit/manager.py @@ -15,7 +15,7 @@ from __future__ import annotations import logging -from pathlib import Path +from pathlib import PurePosixPath from typing import List, Optional from click import ClickException @@ -24,17 +24,22 @@ MissingConnectionRegionError, make_snowsight_url, ) +from snowflake.cli._plugins.nativeapp.artifacts import BundleMap, symlink_or_copy from snowflake.cli._plugins.object.manager import ObjectManager from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli._plugins.streamlit.streamlit_entity_model import ( StreamlitEntityModel, ) +from snowflake.cli._plugins.streamlit.streamlit_project_paths import ( + StreamlitProjectPaths, +) from snowflake.cli.api.commands.experimental_behaviour import ( experimental_behaviour_enabled, ) from snowflake.cli.api.console import cli_console from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor from snowflake.connector.errors import ProgrammingError @@ -54,26 +59,43 @@ def share(self, streamlit_name: FQN, to_role: str) -> SnowflakeCursor: def _put_streamlit_files( self, - root_location: str, - artifacts: Optional[List[Path]] = None, + streamlit_project_paths: StreamlitProjectPaths, + stage_root: str, + artifacts: Optional[List[PathMapping]] = None, ): - cli_console.step(f"Deploying files to {root_location}") + cli_console.step(f"Deploying files to {stage_root}") if not artifacts: return stage_manager = StageManager() - for file in artifacts: - if file.is_dir(): - if not any(file.iterdir()): - cli_console.warning(f"Skipping empty directory: {file}") - continue + bundle_map = BundleMap( + project_root=streamlit_project_paths.project_root, + deploy_root=streamlit_project_paths.deploy_root, + ) + for artifact in artifacts: + bundle_map.add(PathMapping(src=str(artifact.src), dest=artifact.dest)) + + # Clean up deploy root + streamlit_project_paths.remove_up_deploy_root() + for (absolute_src, absolute_dest) in bundle_map.all_mappings( + absolute=True, expand_directories=True + ): + if absolute_src.is_file(): + symlink_or_copy( + absolute_src, + absolute_dest, + deploy_root=streamlit_project_paths.deploy_root, + ) + # Temporary solution, will be replaced with diff + stage_path = ( + PurePosixPath(absolute_dest) + .relative_to(streamlit_project_paths.deploy_root) + .parent + ) + full_stage_path = f"{stage_root}/{stage_path}".rstrip("/") stage_manager.put( - f"{file.joinpath('*')}", f"{root_location}/{file}", 4, True + local_path=absolute_dest, stage_path=full_stage_path, overwrite=True ) - elif len(file.parts) > 1: - stage_manager.put(file, f"{root_location}/{file.parent}", 4, True) - else: - stage_manager.put(file, root_location, 4, True) def _create_streamlit( self, @@ -120,7 +142,12 @@ def _create_streamlit( self.execute_query("\n".join(query)) - def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False): + def deploy( + self, + streamlit: StreamlitEntityModel, + streamlit_project_paths: StreamlitProjectPaths, + replace: bool = False, + ): streamlit_id = streamlit.fqn.using_connection(self._conn) if ( ObjectManager().object_exists(object_type="streamlit", fqn=streamlit_id) @@ -172,12 +199,13 @@ def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False): embedded_stage_name = f"snow://streamlit/{stage_path}" if use_versioned_stage: # "LIVE" is the only supported version for now, but this may change later. - root_location = f"{embedded_stage_name}/versions/live" + stage_root = f"{embedded_stage_name}/versions/live" else: - root_location = f"{embedded_stage_name}/default_checkout" + stage_root = f"{embedded_stage_name}/default_checkout" self._put_streamlit_files( - root_location, + streamlit_project_paths, + stage_root, streamlit.artifacts, ) else: @@ -194,16 +222,18 @@ def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False): cli_console.step(f"Creating {stage_name} stage") stage_manager.create(fqn=stage_name) - root_location = stage_manager.get_standard_stage_prefix( + stage_root = stage_manager.get_standard_stage_prefix( f"{stage_name}/{streamlit_name_for_root_location}" ) - self._put_streamlit_files(root_location, streamlit.artifacts) + self._put_streamlit_files( + streamlit_project_paths, stage_root, streamlit.artifacts + ) self._create_streamlit( streamlit=streamlit, replace=replace, - from_stage_name=root_location, + from_stage_name=stage_root, experimental=False, ) diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py b/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py index 55068adb5a..0ffcf8f124 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py @@ -13,10 +13,10 @@ # limitations under the License. from __future__ import annotations -from pathlib import Path -from typing import List, Literal, Optional +from typing import Literal, Optional -from pydantic import Field, model_validator +from pydantic import Field, field_validator +from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.entities.common import ( EntityModelBase, ExternalAccessBaseModel, @@ -25,6 +25,7 @@ from snowflake.cli.api.project.schemas.updatable_model import ( DiscriminatorField, ) +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping class StreamlitEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel): @@ -43,24 +44,20 @@ class StreamlitEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBase stage: Optional[str] = Field( title="Stage in which the app’s artifacts will be stored", default="streamlit" ) - # Possibly can be PathMapping - artifacts: Optional[List[Path]] = Field( + artifacts: Optional[Artifacts] = Field( title="List of files which should be deployed. Each file needs to exist locally. " "Main file needs to be included in the artifacts.", default=None, ) - @model_validator(mode="after") - def artifacts_must_exists(self): - if not self.artifacts: - return self - - for artifact in self.artifacts: - if "*" in artifact.name: - continue - if not artifact.exists(): - raise ValueError( - f"Specified artifact {artifact} does not exist locally." - ) - - return self + @field_validator("artifacts") + @classmethod + def _convert_artifacts(cls, artifacts: Artifacts) -> Artifacts: + _artifacts = [] + for artifact in artifacts: + if isinstance(artifact, PathMapping): + path_mapping = artifact + else: + path_mapping = PathMapping(src=artifact) + _artifacts.append(path_mapping) + return _artifacts diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py new file mode 100644 index 0000000000..fbe89959b2 --- /dev/null +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from dataclasses import dataclass + +from snowflake.cli.api.project.project_paths import ProjectPaths + + +@dataclass +class StreamlitProjectPaths(ProjectPaths): + """ + This class allows you to manage files paths related to given project. + """ diff --git a/src/snowflake/cli/api/feature_flags.py b/src/snowflake/cli/api/feature_flags.py index 2a56458083..8e2907d4af 100644 --- a/src/snowflake/cli/api/feature_flags.py +++ b/src/snowflake/cli/api/feature_flags.py @@ -64,3 +64,4 @@ class FeatureFlag(FeatureFlagMixin): "ENABLE_STREAMLIT_VERSIONED_STAGE", False ) ENABLE_SPCS_LOG_STREAMING = BooleanFlag("ENABLE_SPCS_LOG_STREAMING", False) + ENABLE_SNOWPARK_GLOB_SUPPORT = BooleanFlag("ENABLE_SNOWPARK_GLOB_SUPPORT", False) diff --git a/src/snowflake/cli/api/project/definition_conversion.py b/src/snowflake/cli/api/project/definition_conversion.py index 7834586a07..b823580e24 100644 --- a/src/snowflake/cli/api/project/definition_conversion.py +++ b/src/snowflake/cli/api/project/definition_conversion.py @@ -222,10 +222,11 @@ def convert_streamlit_to_v2_data(streamlit: Streamlit) -> Dict[str, Any]: environment_file, pages_dir, ] - artifacts = [a for a in artifacts if a is not None] + artifacts = [str(a) for a in artifacts if a is not None] if streamlit.additional_source_files: - artifacts.extend(streamlit.additional_source_files) + for additional_file in streamlit.additional_source_files: + artifacts.append(str(additional_file)) identifier = {"name": streamlit.name} if streamlit.schema_name: diff --git a/src/snowflake/cli/api/project/project_paths.py b/src/snowflake/cli/api/project/project_paths.py new file mode 100644 index 0000000000..f6b2a14d99 --- /dev/null +++ b/src/snowflake/cli/api/project/project_paths.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from pathlib import Path +from shutil import rmtree + + +@dataclass +class ProjectPaths: + project_root: Path + + @property + def deploy_root(self) -> Path: + return self.project_root / "output" + + def remove_up_deploy_root(self) -> None: + if self.deploy_root.exists(): + rmtree(self.deploy_root) diff --git a/src/snowflake/cli/api/project/schemas/commons.py b/src/snowflake/cli/api/project/schemas/commons.py new file mode 100644 index 0000000000..b512afbb39 --- /dev/null +++ b/src/snowflake/cli/api/project/schemas/commons.py @@ -0,0 +1,5 @@ +from typing import List, Union + +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping + +Artifacts = List[Union[PathMapping, str]] diff --git a/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py b/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py index 2123dcacf1..d0c065c7f3 100644 --- a/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py +++ b/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py @@ -15,9 +15,10 @@ from __future__ import annotations import re -from typing import List, Optional, Union +from typing import List, Optional from pydantic import Field, field_validator +from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.updatable_model import UpdatableModel from snowflake.cli.api.project.schemas.v1.native_app.application import ( Application, @@ -34,7 +35,7 @@ class NativeApp(UpdatableModel): name: str = Field( title="Project identifier", ) - artifacts: List[Union[PathMapping, str]] = Field( + artifacts: Artifacts = Field( title="List of file source and destination pairs to add to the deploy root", ) bundle_root: Optional[str] = Field( @@ -69,10 +70,8 @@ def validate_source_stage(cls, input_value: str): @field_validator("artifacts") @classmethod - def transform_artifacts( - cls, orig_artifacts: List[Union[PathMapping, str]] - ) -> List[PathMapping]: - transformed_artifacts = [] + def transform_artifacts(cls, orig_artifacts: Artifacts) -> List[PathMapping]: + transformed_artifacts: List[PathMapping] = [] if orig_artifacts is None: return transformed_artifacts diff --git a/tests/helpers/__snapshots__/test_v1_to_v2.ambr b/tests/helpers/__snapshots__/test_v1_to_v2.ambr index 2e7458624b..d1e6e41d71 100644 --- a/tests/helpers/__snapshots__/test_v1_to_v2.ambr +++ b/tests/helpers/__snapshots__/test_v1_to_v2.ambr @@ -115,10 +115,10 @@ pages_dir: pages stage: artifacts: - - streamlit_app.py - - environment.yml - - pages - - common/hello.py + - src: streamlit_app.py + - src: environment.yml + - src: pages + - src: common/hello.py env: streamlit_title: My Fancy Streamlit @@ -388,7 +388,7 @@ type: string stage: dev_deployment artifacts: - - src: app + - src: app/ dest: my_snowpark_project type: procedure execute_as_caller: false @@ -412,7 +412,7 @@ runtime: '3.10' stage: dev_deployment artifacts: - - src: app + - src: app/ dest: my_snowpark_project type: function test_streamlit: @@ -425,9 +425,9 @@ pages_dir: None stage: streamlit artifacts: - - streamlit_app.py - - environment.yml - - pages + - src: streamlit_app.py + - src: environment.yml + - src: pages pkg: meta: role: pkg_role diff --git a/tests/project/test_project_definition_v2.py b/tests/project/test_project_definition_v2.py index f54497ae74..e639f41cfa 100644 --- a/tests/project/test_project_definition_v2.py +++ b/tests/project/test_project_definition_v2.py @@ -375,7 +375,7 @@ def test_v1_to_v2_conversion( ) artifact = PathMapping( - src=Path(definition_v1.snowpark.src), + src=definition_v1.snowpark.src, dest=definition_v1.snowpark.project_name, ) for v1_procedure in definition_v1.snowpark.procedures: diff --git a/tests/snowpark/__snapshots__/test_function_old_build.ambr b/tests/snowpark/__snapshots__/test_function_old_build.ambr new file mode 100644 index 0000000000..edde16c81f --- /dev/null +++ b/tests/snowpark/__snapshots__/test_function_old_build.ambr @@ -0,0 +1,62 @@ +# serializer version: 1 +# name: test_deploy_function_fully_qualified_name[ok] + ''' + Performing initial validation + Checking remote state + Preparing required stages and artifacts + Creating (if not exists) stage: dev_deployment + Uploading app.zip to @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ + Creating Snowpark entities + Creating function custom_db.custom_schema.fqn_function + Creating function custom_schema.fqn_function_only_schema + Creating function custom_schema.schema_function + Creating function custom_db.PUBLIC.database_function + Creating function custom_db.custom_schema.database_function + Creating function custom_database.custom_schema.fqn_function3 + +------------------------------------------------------------------------------+ + | object | type | status | + |---------------------------------------------------------+----------+---------| + | custom_db.custom_schema.fqn_function(name string) | function | created | + | MockDatabase.custom_schema.fqn_function_only_schema(nam | function | created | + | e string) | | | + | MockDatabase.custom_schema.schema_function(name string) | function | created | + | custom_db.MockSchema.database_function(name string) | function | created | + | custom_db.custom_schema.database_function(name string) | function | created | + | custom_database.custom_schema.fqn_function3(name | function | created | + | string) | | | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_function_fully_qualified_name_duplicated_database[database error] + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | Database provided but name | + | 'custom_database.custom_schema.fqn_function_error' is fully qualified name. | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_function_fully_qualified_name_duplicated_schema[schema error] + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | Schema provided but name 'custom_schema.fqn_function_error' is fully | + | qualified name. | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_function_secrets_without_external_access + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | func1 defined with secrets but without external integration. | + +------------------------------------------------------------------------------+ + + ''' +# --- diff --git a/tests/snowpark/__snapshots__/test_models.ambr b/tests/snowpark/__snapshots__/test_models.ambr new file mode 100644 index 0000000000..9564dc9303 --- /dev/null +++ b/tests/snowpark/__snapshots__/test_models.ambr @@ -0,0 +1,14 @@ +# serializer version: 1 +# name: test_raise_error_when_artifact_contains_asterix + ''' + +- Error ----------------------------------------------------------------------+ + | During evaluation of DefinitionV20 in project definition following errors | + | were encountered: | + | For field entities.hello_procedure.procedure.artifacts you provided | + | '['src/*']'. This caused: Value error, If you want to use glob patterns in | + | artifacts, you need to enable the Snowpark new build feature flag | + | (ENABLE_SNOWPARK_GLOB_SUPPORT=true) | + +------------------------------------------------------------------------------+ + + ''' +# --- diff --git a/tests/snowpark/__snapshots__/test_procedure_old_build.ambr b/tests/snowpark/__snapshots__/test_procedure_old_build.ambr new file mode 100644 index 0000000000..aab627199a --- /dev/null +++ b/tests/snowpark/__snapshots__/test_procedure_old_build.ambr @@ -0,0 +1,56 @@ +# serializer version: 1 +# name: test_deploy_procedure_fails_if_integration_does_not_exists + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | Following external access integration does not exists in Snowflake: | + | external_2 | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_procedure_fails_if_object_exists_and_no_replace + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | Following objects already exists. Consider using --replace. | + | procedure: procedureName | + | procedure: test | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_procedure_fully_qualified_name[database error] + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | Database provided but name | + | 'custom_database.custom_schema.fqn_procedure_error' is fully qualified name. | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_procedure_fully_qualified_name_duplicated_schema[schema error] + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | Schema provided but name 'custom_schema.fqn_procedure_error' is fully | + | qualified name. | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_deploy_procedure_secrets_without_external_access + ''' + Performing initial validation + Checking remote state + +- Error ----------------------------------------------------------------------+ + | procedureName defined with secrets but without external integration. | + +------------------------------------------------------------------------------+ + + ''' +# --- diff --git a/tests/snowpark/test_artifacts.py b/tests/snowpark/test_artifacts.py new file mode 100644 index 0000000000..c573ab9b6d --- /dev/null +++ b/tests/snowpark/test_artifacts.py @@ -0,0 +1,215 @@ +import os +from pathlib import Path +from unittest import mock + +import pytest +from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED +from snowflake.connector import ProgrammingError +from snowflake.connector.compat import IS_WINDOWS + +mock_session_has_warehouse = mock.patch( + "snowflake.cli.api.sql_execution.SqlExecutionMixin.session_has_warehouse", + lambda _: True, +) + + +@pytest.mark.parametrize( + "artifacts, local_path, stage_path", + [ + ("src", Path("output") / "src.zip", "/"), + ("src/", Path("output") / "src.zip", "/"), + ("src/*", Path("output") / "src.zip", "/"), + ("src/*.py", Path("output") / "src.zip", "/"), + ( + "src/dir/dir_app.py", + Path("output") / "src" / "dir" / "dir_app.py", + "/src/dir/", + ), + ( + {"src": "src/**/*", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src/", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src/*", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, + "output/source/dir/apps/dir_app.py", + "/source/dir/apps/", + ), + ], +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put") +@mock_session_has_warehouse +def test_build_and_deploy_with_artifacts( + mock_sm_put, + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + artifacts, + local_path, + stage_path, + enable_snowpark_glob_support_feature_flag, +): + mock_om_describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("glob_patterns") as tmp: + alter_snowflake_yml( + tmp / "snowflake.yml", "entities.hello_procedure.artifacts", [artifacts] + ) + + result = runner.invoke( + [ + "snowpark", + "build", + ] + ) + assert result.exit_code == 0, result.output + + result = runner.invoke( + [ + "snowpark", + "deploy", + ] + ) + assert result.exit_code == 0, result.output + # Windows needs absolute paths. + if IS_WINDOWS: + tmp_path = tmp.absolute() + else: + tmp_path = tmp.resolve() + assert { + "local_path": tmp_path / local_path, + "stage_path": "@MockDatabase.MockSchema.dev_deployment" + stage_path, + } in _extract_put_calls(mock_sm_put) + + +@pytest.mark.parametrize( + "artifact, local_path, stage_path", + [ + ("src", Path("output") / "src.zip", "/"), + ("src/", Path("output") / "src.zip", "/"), + ("src/*", Path("output") / "src.zip", "/"), + ("src/*.py", Path("output") / "src.zip", "/"), + ( + "src/dir/dir_app.py", + Path("output") / "src" / "dir" / "dir_app.py", + "/src/dir/", + ), + ( + {"src": "src/**/*", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src/", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src/*", "dest": "source/"}, + Path("output") / "source" / "src.zip", + "/source/", + ), + ( + {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, + Path("output") / "source" / "dir" / "apps" / "dir_app.py", + "/source/dir/apps/", + ), + ], +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put") +@mock_session_has_warehouse +def test_build_and_deploy_with_artifacts_run_from_other_directory( + mock_sm_put, + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + artifact, + local_path, + stage_path, + enable_snowpark_glob_support_feature_flag, +): + mock_om_describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("glob_patterns") as tmp: + os.chdir(Path(os.getcwd()).parent) + alter_snowflake_yml( + tmp / "snowflake.yml", "entities.hello_procedure.artifacts", [artifact] + ) + + result = runner.invoke( + [ + "snowpark", + "build", + "-p", + tmp, + ] + ) + assert result.exit_code == 0, result.output + + result = runner.invoke( + [ + "snowpark", + "deploy", + "-p", + tmp, + ] + ) + assert result.exit_code == 0, result.output + assert { + "local_path": tmp / local_path, + "stage_path": "@MockDatabase.MockSchema.dev_deployment" + stage_path, + } in _extract_put_calls(mock_sm_put) + + +def _extract_put_calls(mock_sm_put): + # Extract the put calls from the mock for better visibility in test logs + return [ + { + "local_path": call.kwargs.get("local_path"), + "stage_path": call.kwargs.get("stage_path"), + } + for call in mock_sm_put.mock_calls + if call.kwargs.get("local_path") + ] diff --git a/tests/snowpark/test_build.py b/tests/snowpark/test_build.py index 008a832dd1..9298ed2b32 100644 --- a/tests/snowpark/test_build.py +++ b/tests/snowpark/test_build.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from typing import Set from unittest.mock import patch +from zipfile import ZipFile +import pytest from snowflake.cli._plugins.snowpark.package_utils import ( DownloadUnavailablePackagesResult, ) @@ -28,3 +30,37 @@ def test_snowpark_build_no_deprecated_warnings_by_default( result = runner.invoke(["snowpark", "build", "--ignore-anaconda"]) assert result.exit_code == 0, result.output assert "flag is deprecated" not in result.output + + +@pytest.mark.parametrize( + "artifacts, zip_name, expected_files", + [ + ("src", "src.zip", {"app.py", "dir/dir_app.py"}), + ("src/", "src.zip", {"app.py", "dir/dir_app.py"}), + ("src/*", "src.zip", {"app.py", "dir/dir_app.py"}), + ("src/*.py", "src.zip", {"app.py"}), + ("src/**/*.py", "src.zip", {"app.py", "dir/dir_app.py"}), + ], +) +def test_build_with_glob_patterns_in_artifacts( + runner, + enable_snowpark_glob_support_feature_flag, + project_directory, + alter_snowflake_yml, + artifacts, + zip_name, + expected_files, +): + with project_directory("glob_patterns") as tmp_dir: + alter_snowflake_yml( + tmp_dir / "snowflake.yml", "entities.hello_procedure.artifacts", [artifacts] + ) + + result = runner.invoke(["snowpark", "build", "--ignore-anaconda"]) + assert result.exit_code == 0, result.output + _assert_zip_contains(tmp_dir / "output" / zip_name, expected_files) + + +def _assert_zip_contains(app_zip: str, expected_files: Set[str]): + zip_file = ZipFile(app_zip) + assert set(zip_file.namelist()) == expected_files diff --git a/tests/snowpark/test_function.py b/tests/snowpark/test_function.py index 180c3a9806..a2887da832 100644 --- a/tests/snowpark/test_function.py +++ b/tests/snowpark/test_function.py @@ -18,6 +18,9 @@ from unittest import mock import pytest +from snowflake.cli._plugins.snowpark.package_utils import ( + DownloadUnavailablePackagesResult, +) from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED from snowflake.connector import ProgrammingError @@ -155,6 +158,7 @@ def test_deploy_function_secrets_without_external_access( project_directory, os_agnostic_snapshot, project_name, + enable_snowpark_glob_support_feature_flag, ): mock_object_manager.return_value.show.return_value = [ {"name": "external_1", "type": "EXTERNAL_ACCESS"}, @@ -164,6 +168,15 @@ def test_deploy_function_secrets_without_external_access( mock_conn.return_value = ctx with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( [ "snowpark", @@ -179,15 +192,21 @@ def test_deploy_function_secrets_without_external_access( "project_name", ["snowpark_functions", "snowpark_functions_v2"] ) @mock.patch("snowflake.connector.connect") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_no_changes( + mock_download, mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() rows = [ ("packages", '["foo==1.2.3", "bar>=3.0.0"]'), ("handler", "app.func1_handler"), @@ -217,7 +236,7 @@ def test_deploy_function_no_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", ] @@ -225,10 +244,14 @@ def test_deploy_function_no_changes( "project_name", ["snowpark_functions", "snowpark_functions_v2"] ) @mock.patch("snowflake.connector.connect") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_needs_update_because_packages_changes( - mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name + mock_download, mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name, enable_snowpark_glob_support_feature_flag ): + mock_download.return_value = DownloadUnavailablePackagesResult() rows = [ ("packages", '["foo==1.2.3"]'), ("handler", "main.py:app"), @@ -256,7 +279,7 @@ def test_deploy_function_needs_update_because_packages_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", dedent( """\ create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) @@ -276,10 +299,14 @@ def test_deploy_function_needs_update_because_packages_changes( "project_name", ["snowpark_functions", "snowpark_functions_v2"] ) @mock.patch("snowflake.connector.connect") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_needs_update_because_handler_changes( - mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name + mock_download, mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name, enable_snowpark_glob_support_feature_flag ): + mock_download.return_value = DownloadUnavailablePackagesResult() rows = [ ("packages", '["foo==1.2.3", "bar>=3.0.0"]'), ("handler", "main.py:oldApp"), @@ -307,7 +334,7 @@ def test_deploy_function_needs_update_because_handler_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" f" auto_compress=false parallel=4 overwrite=True", dedent( """\ @@ -334,8 +361,12 @@ def test_deploy_function_needs_update_because_handler_changes( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_fully_qualified_name_duplicated_database( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -345,7 +376,9 @@ def test_deploy_function_fully_qualified_name_duplicated_database( alter_snowflake_yml, os_agnostic_snapshot, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() number_of_functions_in_project = 6 mock_om_describe.side_effect = [ ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), @@ -353,7 +386,16 @@ def test_deploy_function_fully_qualified_name_duplicated_database( ctx = mock_ctx() mock_conn.return_value = ctx - with project_directory(project_name) as tmp_dir: + with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy"]) assert result.output == os_agnostic_snapshot(name="database error") @@ -371,8 +413,12 @@ def test_deploy_function_fully_qualified_name_duplicated_database( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_fully_qualified_name_duplicated_schema( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -383,7 +429,9 @@ def test_deploy_function_fully_qualified_name_duplicated_schema( os_agnostic_snapshot, project_name, path_in_project_file, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() number_of_functions_in_project = 6 mock_om_describe.side_effect = [ ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), @@ -397,6 +445,15 @@ def test_deploy_function_fully_qualified_name_duplicated_schema( parameter_path=path_in_project_file, value="custom_schema.fqn_function_error", ) + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy"]) assert result.output == os_agnostic_snapshot(name="schema error") @@ -414,8 +471,12 @@ def test_deploy_function_fully_qualified_name_duplicated_schema( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_fully_qualified_name( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -426,7 +487,9 @@ def test_deploy_function_fully_qualified_name( os_agnostic_snapshot, project_name, parameter_path, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() number_of_functions_in_project = 6 mock_om_describe.side_effect = [ ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), @@ -440,6 +503,15 @@ def test_deploy_function_fully_qualified_name( parameter_path=parameter_path, value="fqn_function3", ) + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy"]) assert result.exit_code == 0 assert result.output == os_agnostic_snapshot(name="ok") @@ -472,8 +544,12 @@ def test_deploy_function_fully_qualified_name( ) @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_function_with_empty_default_value( + mock_download, mock_object_manager, mock_connector, mock_ctx, @@ -485,7 +561,9 @@ def test_deploy_function_with_empty_default_value( project_name, signature_path, runtime_path, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_object_manager.return_value.describe.side_effect = ProgrammingError( errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED ) @@ -504,6 +582,15 @@ def test_deploy_function_with_empty_default_value( parameter_path=runtime_path, value="3.10", ) + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( ["snowpark", "deploy", "--format", "json"], catch_exceptions=False ) @@ -566,6 +653,15 @@ def _deploy_function( (Path(temp_dir) / "requirements.snowflake.txt").write_text( "foo==1.2.3\nbar>=3.0.0" ) + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( [ "snowpark", diff --git a/tests/snowpark/test_function_old_build.py b/tests/snowpark/test_function_old_build.py new file mode 100644 index 0000000000..2a78955d82 --- /dev/null +++ b/tests/snowpark/test_function_old_build.py @@ -0,0 +1,506 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path +from textwrap import dedent +from unittest import mock + +import pytest +from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED +from snowflake.connector import ProgrammingError + +from tests_common import IS_WINDOWS + +if IS_WINDOWS: + pytest.skip("Requires further refactor to work on Windows", allow_module_level=True) + + +mock_session_has_warehouse = mock.patch( + "snowflake.cli.api.sql_execution.SqlExecutionMixin.session_has_warehouse", + lambda _: True, +) + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager") +@mock_session_has_warehouse +def test_deploy_function( + mock_object_manager, + mock_connector, + mock_ctx, + runner, + project_directory, +): + mock_object_manager.return_value.describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_connector.return_value = ctx + with project_directory("snowpark_functions") as project_dir: + result = runner.invoke( + [ + "snowpark", + "deploy", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f" auto_compress=false parallel=4 overwrite=True", + dedent( + """\ + create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='app.func1_handler' + packages=() + """ + ).strip(), + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager") +@mock_session_has_warehouse +def test_deploy_function_with_external_access( + mock_object_manager, + mock_connector, + mock_ctx, + runner, + project_directory, +): + mock_object_manager.return_value.show.return_value = [ + {"name": "external_1", "type": "EXTERNAL_ACCESS"}, + {"name": "external_2", "type": "EXTERNAL_ACCESS"}, + ] + mock_object_manager.return_value.describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_connector.return_value = ctx + + with project_directory("snowpark_function_external_access") as project_dir: + result = runner.invoke( + [ + "snowpark", + "deploy", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f" auto_compress=false parallel=4 overwrite=True", + dedent( + """\ + create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string, b variant) + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='app.func1_handler' + packages=() + external_access_integrations=(external_1, external_2) + secrets=('cred'=cred_name, 'other'=other_name) + """ + ).strip(), + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager") +@mock_session_has_warehouse +def test_deploy_function_secrets_without_external_access( + mock_object_manager, + mock_conn, + runner, + mock_ctx, + project_directory, + os_agnostic_snapshot, +): + mock_object_manager.return_value.show.return_value = [ + {"name": "external_1", "type": "EXTERNAL_ACCESS"}, + {"name": "external_2", "type": "EXTERNAL_ACCESS"}, + ] + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_function_secrets_without_external_access"): + result = runner.invoke( + [ + "snowpark", + "deploy", + ], + ) + + assert result.exit_code == 1, result.output + assert result.output == os_agnostic_snapshot + + +@mock.patch("snowflake.connector.connect") +@mock_session_has_warehouse +def test_deploy_function_no_changes( + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, +): + rows = [ + ("packages", '["foo==1.2.3", "bar>=3.0.0"]'), + ("handler", "app.func1_handler"), + ("returns", "string"), + ("imports", "dev_deployment/my_snowpark_project/app.py"), + ("runtime_version", "3.10"), + ] + + queries, result, project_dir = _deploy_function( + rows, + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, + "--replace", + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [ + { + "object": "MockDatabase.MockSchema.func1(a string default 'default value', b variant)", + "status": "packages updated", + "type": "function", + } + ] + assert queries == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + ] + + +@mock.patch("snowflake.connector.connect") +@mock_session_has_warehouse +def test_deploy_function_needs_update_because_packages_changes( + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, +): + rows = [ + ("packages", '["foo==1.2.3"]'), + ("handler", "main.py:app"), + ("returns", "table(variant)"), + ] + + queries, result, project_dir = _deploy_function( + rows, + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, + "--replace", + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [ + { + "object": "MockDatabase.MockSchema.func1(a string default 'default value', b variant)", + "status": "definition updated", + "type": "function", + } + ] + assert queries == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + dedent( + """\ + create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='app.func1_handler' + packages=('foo==1.2.3','bar>=3.0.0') + """ + ).strip(), + ] + + +@mock.patch("snowflake.connector.connect") +@mock_session_has_warehouse +def test_deploy_function_needs_update_because_handler_changes( + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, +): + rows = [ + ("packages", '["foo==1.2.3", "bar>=3.0.0"]'), + ("handler", "main.py:oldApp"), + ("returns", "table(variant)"), + ] + + queries, result, project_dir = _deploy_function( + rows, + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, + "--replace", + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [ + { + "object": "MockDatabase.MockSchema.func1(a string default 'default value', b variant)", + "status": "definition updated", + "type": "function", + } + ] + assert queries == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f" auto_compress=false parallel=4 overwrite=True", + dedent( + """\ + create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='app.func1_handler' + packages=('foo==1.2.3','bar>=3.0.0') + """ + ).strip(), + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_function_fully_qualified_name_duplicated_database( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + os_agnostic_snapshot, +): + number_of_functions_in_project = 6 + mock_om_describe.side_effect = [ + ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), + ] * number_of_functions_in_project + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_function_fully_qualified_name") as tmp_dir: + result = runner.invoke(["snowpark", "deploy"]) + assert result.output == os_agnostic_snapshot(name="database error") + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_function_fully_qualified_name_duplicated_schema( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + os_agnostic_snapshot, +): + number_of_functions_in_project = 6 + mock_om_describe.side_effect = [ + ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), + ] * number_of_functions_in_project + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_function_fully_qualified_name") as tmp_dir: + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.functions.5.name", + value="custom_schema.fqn_function_error", + ) + result = runner.invoke(["snowpark", "deploy"]) + assert result.output == os_agnostic_snapshot(name="schema error") + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_function_fully_qualified_name( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + os_agnostic_snapshot, +): + number_of_functions_in_project = 6 + mock_om_describe.side_effect = [ + ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), + ] * number_of_functions_in_project + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_function_fully_qualified_name") as tmp_dir: + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.functions.5.name", + value="fqn_function3", + ) + result = runner.invoke(["snowpark", "deploy"]) + assert result.exit_code == 0 + assert result.output == os_agnostic_snapshot(name="ok") + + +@pytest.mark.parametrize( + "parameter_type,default_value", + [ + ("string", None), + ("string", ""), + ("int", None), + ("variant", None), + ("bool", None), + ], +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager") +@mock_session_has_warehouse +def test_deploy_function_with_empty_default_value( + mock_object_manager, + mock_connector, + mock_ctx, + runner, + project_directory, + alter_snowflake_yml, + parameter_type, + default_value, +): + mock_object_manager.return_value.describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_connector.return_value = ctx + with project_directory("snowpark_functions") as project_dir: + snowflake_yml = project_dir / "snowflake.yml" + for param, value in [("type", parameter_type), ("default", default_value)]: + alter_snowflake_yml( + snowflake_yml, + parameter_path=f"snowpark.functions.0.signature.0.{param}", + value=value, + ) + alter_snowflake_yml( + snowflake_yml, + parameter_path=f"snowpark.functions.0.runtime", + value="3.10", + ) + result = runner.invoke( + ["snowpark", "deploy", "--format", "json"], catch_exceptions=False + ) + default_value_json = default_value + if default_value is None: + default_value_json = "null" + elif parameter_type == "string": + default_value_json = f"'{default_value}'" + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [ + { + "object": f"MockDatabase.MockSchema.func1(a {parameter_type} default {default_value_json}, b variant)", + "status": "created", + "type": "function", + } + ] + + +@mock.patch("snowflake.connector.connect") +def test_execute_function(mock_connector, runner, mock_ctx): + ctx = mock_ctx() + mock_connector.return_value = ctx + result = runner.invoke( + [ + "snowpark", + "execute", + "function", + "functionName(42, 'string')", + ] + ) + + assert result.exit_code == 0, result.output + assert ctx.get_query() == "select functionName(42, 'string')" + + +def _deploy_function( + rows, + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, + *args, +): + ctx = mock_ctx(mock_cursor(rows=rows, columns=[])) + mock_connector.return_value = ctx + with ( + mock.patch( + "snowflake.cli._plugins.snowpark.commands.ObjectManager.describe" + ) as om_describe, + mock.patch( + "snowflake.cli._plugins.snowpark.commands.ObjectManager.show" + ) as om_show, + ): + om_describe.return_value = rows + + with project_directory("snowpark_functions") as temp_dir: + (Path(temp_dir) / "requirements.snowflake.txt").write_text( + "foo==1.2.3\nbar>=3.0.0" + ) + result = runner.invoke( + [ + "snowpark", + "deploy", + "--format", + "json", + *args, + ] + ) + queries = ctx.get_queries() + return queries, result, temp_dir diff --git a/tests/snowpark/test_models.py b/tests/snowpark/test_models.py index c1e847398a..b18dd9198c 100644 --- a/tests/snowpark/test_models.py +++ b/tests/snowpark/test_models.py @@ -86,3 +86,17 @@ def test_wheel_metadata_parsing(test_root_path): assert meta.name == "zendesk" assert meta.wheel_path == wheel_path.path assert meta.dependencies == ["httplib2", "simplejson"] + + +def test_raise_error_when_artifact_contains_asterix( + runner, project_directory, alter_snowflake_yml, os_agnostic_snapshot +): + with project_directory("glob_patterns") as tmp_dir: + alter_snowflake_yml( + tmp_dir / "snowflake.yml", "entities.hello_procedure.artifacts", ["src/*"] + ) + + result = runner.invoke(["snowpark", "build"]) + + assert result.exit_code == 1 + assert result.output == os_agnostic_snapshot diff --git a/tests/snowpark/test_procedure.py b/tests/snowpark/test_procedure.py index cefef10c29..3b2069e7fd 100644 --- a/tests/snowpark/test_procedure.py +++ b/tests/snowpark/test_procedure.py @@ -19,6 +19,9 @@ from unittest.mock import call import pytest +from snowflake.cli._plugins.snowpark.package_utils import ( + DownloadUnavailablePackagesResult, +) from snowflake.cli.api.constants import ObjectType from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED from snowflake.cli.api.identifiers import FQN @@ -58,8 +61,12 @@ def test_deploy_function_no_procedure(runner, project_directory): @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -67,8 +74,9 @@ def test_deploy_procedure( mock_ctx, project_directory, project_name, + enable_snowpark_glob_support_feature_flag, ): - + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.side_effect = ProgrammingError( errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED ) @@ -76,6 +84,15 @@ def test_deploy_procedure( mock_conn.return_value = ctx with project_directory(project_name) as tmp: + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( [ "snowpark", @@ -92,7 +109,7 @@ def test_deploy_procedure( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(tmp).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + f"put file://{Path(tmp).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", dedent( """\ create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) @@ -127,8 +144,12 @@ def test_deploy_procedure( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_with_external_access( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -136,7 +157,9 @@ def test_deploy_procedure_with_external_access( mock_ctx, project_directory, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.side_effect = ProgrammingError( errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED ) @@ -149,6 +172,15 @@ def test_deploy_procedure_with_external_access( mock_conn.return_value = ctx with project_directory(project_name) as project_dir: + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( [ "snowpark", @@ -167,7 +199,7 @@ def test_deploy_procedure_with_external_access( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" f" auto_compress=false parallel=4 overwrite=True", dedent( """\ @@ -196,8 +228,12 @@ def test_deploy_procedure_with_external_access( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_secrets_without_external_access( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -206,7 +242,9 @@ def test_deploy_procedure_secrets_without_external_access( project_directory, os_agnostic_snapshot, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() ctx = mock_ctx() mock_conn.return_value = ctx @@ -216,6 +254,15 @@ def test_deploy_procedure_secrets_without_external_access( ] with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( [ "snowpark", @@ -235,8 +282,12 @@ def test_deploy_procedure_secrets_without_external_access( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_fails_if_integration_does_not_exists( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -245,7 +296,9 @@ def test_deploy_procedure_fails_if_integration_does_not_exists( project_directory, os_agnostic_snapshot, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() ctx = mock_ctx() mock_conn.return_value = ctx @@ -254,6 +307,15 @@ def test_deploy_procedure_fails_if_integration_does_not_exists( ] with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke( [ "snowpark", @@ -275,8 +337,12 @@ def test_deploy_procedure_fails_if_integration_does_not_exists( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_fails_if_object_exists_and_no_replace( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -287,7 +353,9 @@ def test_deploy_procedure_fails_if_object_exists_and_no_replace( project_directory, os_agnostic_snapshot, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.return_value = mock_cursor( [ ("packages", "[]"), @@ -300,6 +368,15 @@ def test_deploy_procedure_fails_if_object_exists_and_no_replace( mock_conn.return_value = ctx with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy"]) assert result.exit_code == 1 @@ -312,8 +389,12 @@ def test_deploy_procedure_fails_if_object_exists_and_no_replace( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_replace_nothing_to_update( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -323,7 +404,9 @@ def test_deploy_procedure_replace_nothing_to_update( project_directory, caplog, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.side_effect = [ mock_cursor( [ @@ -349,6 +432,15 @@ def test_deploy_procedure_replace_nothing_to_update( mock_conn.return_value = ctx with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy", "--replace", "--format", "json"]) assert result.exit_code == 0, result.output @@ -372,8 +464,12 @@ def test_deploy_procedure_replace_nothing_to_update( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_replace_updates_single_object( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -382,7 +478,9 @@ def test_deploy_procedure_replace_updates_single_object( mock_ctx, project_directory, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.side_effect = [ mock_cursor( [ @@ -407,6 +505,15 @@ def test_deploy_procedure_replace_updates_single_object( mock_conn.return_value = ctx with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy", "--replace", "--format", "json"]) assert result.exit_code == 0 @@ -430,8 +537,12 @@ def test_deploy_procedure_replace_updates_single_object( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_replace_creates_missing_object( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -440,7 +551,9 @@ def test_deploy_procedure_replace_creates_missing_object( mock_ctx, project_directory, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.side_effect = [ mock_cursor( [ @@ -457,9 +570,18 @@ def test_deploy_procedure_replace_creates_missing_object( mock_conn.return_value = ctx with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy", "--replace", "--format", "json"]) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output assert json.loads(result.output) == [ { "object": "MockDatabase.MockSchema.procedureName(name string)", @@ -484,8 +606,12 @@ def test_deploy_procedure_replace_creates_missing_object( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_fully_qualified_name( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -495,7 +621,9 @@ def test_deploy_procedure_fully_qualified_name( alter_snowflake_yml, os_agnostic_snapshot, project_name, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() number_of_procedures_in_projects = 6 mock_om_describe.side_effect = [ ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), @@ -503,7 +631,16 @@ def test_deploy_procedure_fully_qualified_name( ctx = mock_ctx() mock_conn.return_value = ctx - with project_directory(project_name) as tmp_dir: + with project_directory(project_name): + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy"]) assert result.output == os_agnostic_snapshot(name="database error") @@ -521,8 +658,12 @@ def test_deploy_procedure_fully_qualified_name( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_fully_qualified_name_duplicated_schema( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -533,7 +674,9 @@ def test_deploy_procedure_fully_qualified_name_duplicated_schema( os_agnostic_snapshot, project_name, parameter_path, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() number_of_procedures_in_projects = 6 mock_om_describe.side_effect = [ ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), @@ -547,6 +690,15 @@ def test_deploy_procedure_fully_qualified_name_duplicated_schema( parameter_path=parameter_path, value="custom_schema.fqn_procedure_error", ) + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy"]) assert result.output == os_agnostic_snapshot(name="schema error") @@ -564,8 +716,12 @@ def test_deploy_procedure_fully_qualified_name_duplicated_schema( @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") @mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock.patch( + "snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages" +) @mock_session_has_warehouse def test_deploy_procedure_with_empty_default_value( + mock_download, mock_om_show, mock_om_describe, mock_conn, @@ -575,7 +731,9 @@ def test_deploy_procedure_with_empty_default_value( alter_snowflake_yml, parameter_type, default_value, + enable_snowpark_glob_support_feature_flag, ): + mock_download.return_value = DownloadUnavailablePackagesResult() mock_om_describe.side_effect = ProgrammingError( errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED ) @@ -590,6 +748,16 @@ def test_deploy_procedure_with_empty_default_value( parameter_path=f"snowpark.procedures.0.signature.0.{param}", value=value, ) + + result = runner.invoke( + [ + "snowpark", + "build", + "--ignore-anaconda", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output result = runner.invoke(["snowpark", "deploy", "--format", "json"]) default_value_json = default_value diff --git a/tests/snowpark/test_procedure_old_build.py b/tests/snowpark/test_procedure_old_build.py new file mode 100644 index 0000000000..2f1c817576 --- /dev/null +++ b/tests/snowpark/test_procedure_old_build.py @@ -0,0 +1,555 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path +from textwrap import dedent +from unittest import mock +from unittest.mock import call + +import pytest +from snowflake.cli.api.constants import ObjectType +from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED +from snowflake.cli.api.identifiers import FQN +from snowflake.connector import ProgrammingError + +from tests_common import IS_WINDOWS + +if IS_WINDOWS: + pytest.skip("Requires further refactor to work on Windows", allow_module_level=True) + + +mock_session_has_warehouse = mock.patch( + "snowflake.cli.api.sql_execution.SqlExecutionMixin.session_has_warehouse", + lambda _: True, +) + + +@mock_session_has_warehouse +def test_deploy_function_no_procedure(runner, project_directory): + with project_directory("empty_project"): + result = runner.invoke( + [ + "snowpark", + "deploy", + ], + ) + assert result.exit_code == 1 + assert ( + "No procedures or functions were specified in the project definition." + in result.output + ) + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, +): + + mock_om_describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedures") as tmp: + result = runner.invoke( + [ + "snowpark", + "deploy", + ] + ) + + assert result.exit_code == 0, result.output + mock_om_describe.return_value( + [ + call(object_type=str(ObjectType.PROCEDURE), name="procedureName(string)"), + call(object_type=str(ObjectType.PROCEDURE), name="test()"), + ] + ) + assert ctx.get_queries() == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(tmp).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + dedent( + """\ + create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='hello' + packages=() + """ + ).strip(), + dedent( + """\ + create or replace procedure IDENTIFIER('MockDatabase.MockSchema.test')() + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='test' + packages=() + """ + ).strip(), + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_with_external_access( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, +): + mock_om_describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + mock_om_show.return_value = [ + {"name": "external_1", "type": "EXTERNAL_ACCESS"}, + {"name": "external_2", "type": "EXTERNAL_ACCESS"}, + ] + + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedure_external_access") as project_dir: + result = runner.invoke( + [ + "snowpark", + "deploy", + ] + ) + + assert result.exit_code == 0, result.output + mock_om_describe.assert_has_calls( + [ + call( + object_type=str(ObjectType.PROCEDURE), + fqn=FQN.from_string("MockDatabase.MockSchema.procedureName(string)"), + ), + ] + ) + assert ctx.get_queries() == [ + "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", + f"put file://{Path(project_dir).resolve()}/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" + f" auto_compress=false parallel=4 overwrite=True", + dedent( + """\ + create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) + copy grants + returns string + language python + runtime_version=3.10 + imports=('@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/app.py') + handler='app.hello' + packages=() + external_access_integrations=(external_1, external_2) + secrets=('cred'=cred_name, 'other'=other_name) + """ + ).strip(), + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_secrets_without_external_access( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + os_agnostic_snapshot, +): + ctx = mock_ctx() + mock_conn.return_value = ctx + + mock_om_show.return_value = [ + {"name": "external_1", "type": "EXTERNAL_ACCESS"}, + {"name": "external_2", "type": "EXTERNAL_ACCESS"}, + ] + + with project_directory("snowpark_procedure_secrets_without_external_access"): + result = runner.invoke( + [ + "snowpark", + "deploy", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 1, result.output + assert result.output == os_agnostic_snapshot + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_fails_if_integration_does_not_exists( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + os_agnostic_snapshot, +): + ctx = mock_ctx() + mock_conn.return_value = ctx + + mock_om_show.return_value = [ + {"name": "external_1", "type": "EXTERNAL_ACCESS"}, + ] + + with project_directory("snowpark_procedure_external_access"): + result = runner.invoke( + [ + "snowpark", + "deploy", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 1, result.output + assert result.output == os_agnostic_snapshot + + +@mock.patch( + "snowflake.cli._plugins.snowpark.commands._check_if_all_defined_integrations_exists" +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_fails_if_object_exists_and_no_replace( + mock_om_show, + mock_om_describe, + mock_conn, + _, + runner, + mock_cursor, + mock_ctx, + project_directory, + os_agnostic_snapshot, +): + mock_om_describe.return_value = mock_cursor( + [ + ("packages", "[]"), + ("handler", "hello"), + ("returns", "string"), + ], + columns=["key", "value"], + ) + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedures"): + result = runner.invoke(["snowpark", "deploy"]) + + assert result.exit_code == 1 + assert result.output == os_agnostic_snapshot + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_replace_nothing_to_update( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_cursor, + mock_ctx, + project_directory, + caplog, +): + mock_om_describe.side_effect = [ + mock_cursor( + [ + ("packages", "[]"), + ("handler", "hello"), + ("returns", "string"), + ("imports", "dev_deployment/my_snowpark_project/app.py"), + ], + columns=["key", "value"], + ), + mock_cursor( + [ + ("packages", "[]"), + ("handler", "test"), + ("returns", "string"), + ("imports", "dev_deployment/my_snowpark_project/app.py"), + ("runtime_version", "3.10"), + ], + columns=["key", "value"], + ), + ] + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedures"): + result = runner.invoke(["snowpark", "deploy", "--replace", "--format", "json"]) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [ + { + "object": "MockDatabase.MockSchema.procedureName(name string)", + "status": "packages updated", + "type": "procedure", + }, + { + "object": "MockDatabase.MockSchema.test()", + "status": "packages updated", + "type": "procedure", + }, + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_replace_updates_single_object( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_cursor, + mock_ctx, + project_directory, +): + mock_om_describe.side_effect = [ + mock_cursor( + [ + ("packages", "[]"), + ("handler", "hello"), + ("returns", "string"), + ("imports", "dev_deployment/my_snowpark_project/app.py"), + ], + columns=["key", "value"], + ), + mock_cursor( + [ + ("packages", "[]"), + ("handler", "foo"), + ("returns", "string"), + ("imports", "dev_deployment/my_snowpark_project/app.zip"), + ], + columns=["key", "value"], + ), + ] + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedures"): + result = runner.invoke(["snowpark", "deploy", "--replace", "--format", "json"]) + + assert result.exit_code == 0 + assert json.loads(result.output) == [ + { + "object": "MockDatabase.MockSchema.procedureName(name string)", + "status": "packages updated", + "type": "procedure", + }, + { + "object": "MockDatabase.MockSchema.test()", + "status": "definition updated", + "type": "procedure", + }, + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_replace_creates_missing_object( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_cursor, + mock_ctx, + project_directory, +): + mock_om_describe.side_effect = [ + mock_cursor( + [ + ("packages", "[]"), + ("handler", "hello"), + ("returns", "string"), + ("imports", "dev_deployment/my_snowpark_project/app.py"), + ], + columns=["key", "value"], + ), + ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), + ] + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedures"): + result = runner.invoke(["snowpark", "deploy", "--replace", "--format", "json"]) + + assert result.exit_code == 0 + assert json.loads(result.output) == [ + { + "object": "MockDatabase.MockSchema.procedureName(name string)", + "status": "packages updated", + "type": "procedure", + }, + { + "object": "MockDatabase.MockSchema.test()", + "status": "created", + "type": "procedure", + }, + ] + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_fully_qualified_name( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + os_agnostic_snapshot, +): + number_of_procedures_in_projects = 6 + mock_om_describe.side_effect = [ + ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), + ] * number_of_procedures_in_projects + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedure_fully_qualified_name") as tmp_dir: + result = runner.invoke(["snowpark", "deploy"]) + assert result.output == os_agnostic_snapshot(name="database error") + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_fully_qualified_name_duplicated_schema( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + os_agnostic_snapshot, +): + number_of_procedures_in_projects = 6 + mock_om_describe.side_effect = [ + ProgrammingError(errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED), + ] * number_of_procedures_in_projects + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedure_fully_qualified_name") as tmp_dir: + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.procedures.5.name", + value="custom_schema.fqn_procedure_error", + ) + result = runner.invoke(["snowpark", "deploy"]) + assert result.output == os_agnostic_snapshot(name="schema error") + + +@pytest.mark.parametrize( + "parameter_type,default_value", + [ + ("string", None), + ("string", ""), + ("int", None), + ("variant", None), + ("bool", None), + ], +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.describe") +@mock.patch("snowflake.cli._plugins.snowpark.commands.ObjectManager.show") +@mock_session_has_warehouse +def test_deploy_procedure_with_empty_default_value( + mock_om_show, + mock_om_describe, + mock_conn, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + parameter_type, + default_value, +): + mock_om_describe.side_effect = ProgrammingError( + errno=DOES_NOT_EXIST_OR_NOT_AUTHORIZED + ) + ctx = mock_ctx() + mock_conn.return_value = ctx + + with project_directory("snowpark_procedures") as project_dir: + snowflake_yml = project_dir / "snowflake.yml" + for param, value in [("type", parameter_type), ("default", default_value)]: + alter_snowflake_yml( + snowflake_yml, + parameter_path=f"snowpark.procedures.0.signature.0.{param}", + value=value, + ) + result = runner.invoke(["snowpark", "deploy", "--format", "json"]) + + default_value_json = default_value + if default_value is None: + default_value_json = "null" + elif parameter_type == "string": + default_value_json = f"'{default_value}'" + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [ + { + "object": f"MockDatabase.MockSchema.procedureName(name {parameter_type} default {default_value_json})", + "status": "created", + "type": "procedure", + }, + { + "object": "MockDatabase.MockSchema.test()", + "status": "created", + "type": "procedure", + }, + ] diff --git a/tests/snowpark/test_project_paths.py b/tests/snowpark/test_project_paths.py new file mode 100644 index 0000000000..55796e1a98 --- /dev/null +++ b/tests/snowpark/test_project_paths.py @@ -0,0 +1,229 @@ +from pathlib import Path +from unittest import mock + +import pytest +from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact + + +@pytest.mark.parametrize( + "path, dest, is_file, expected_path", + [ + ("src", None, False, "@db.public.stage/src.zip"), + ("src/", None, False, "@db.public.stage/src.zip"), + ("src", "source", False, "@db.public.stage/source/src.zip"), + ("src/app.py", None, True, "@db.public.stage/src/app.py"), + ("src/app.py", "source/new_app.py", True, "@db.public.stage/source/new_app.py"), + ("src/dir/dir2/app.py", None, True, "@db.public.stage/src/dir/dir2/app.py"), + ("src/dir/dir2/app.py", "source/", True, "@db.public.stage/source/app.py"), + ("src/*", "source/", False, "@db.public.stage/source/src.zip"), + ("src/**/*.py", None, False, "@db.public.stage/src.zip"), + ("src/**/*.py", "source/", False, "@db.public.stage/source/src.zip"), + ("src/app*", None, False, "@db.public.stage/src.zip"), + ("src/app[1-5].py", None, False, "@db.public.stage/src.zip"), + ], +) +@mock.patch("snowflake.cli.api.cli_global_context.get_cli_context") +def test_artifact_import_path(mock_ctx_context, path, dest, is_file, expected_path): + mock_connection = mock.Mock() + mock_connection.database = "db" + mock_connection.schema = "public" + mock_ctx_context.return_value.connection = mock_connection + stage = "stage" + + with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): + import_path = Artefact(Path(), Path(path), dest).import_path(stage) + + assert import_path == expected_path + + +@pytest.mark.parametrize( + "path, dest, is_file, expected_path", + [ + ("src", None, False, "@db.public.stage/"), + ("src/", None, False, "@db.public.stage/"), + ("src", "source", False, "@db.public.stage/source/"), + ("src/app.py", None, True, "@db.public.stage/src/"), + ("src/app.py", "source/new_app.py", True, "@db.public.stage/source/"), + ("src/dir/dir2/app.py", None, True, "@db.public.stage/src/dir/dir2/"), + ("src/dir/dir2/app.py", "source/", True, "@db.public.stage/source/"), + ("src/*", "source/", False, "@db.public.stage/source/"), + ("src/**/*.py", None, False, "@db.public.stage/"), + ("src/**/*.py", "source/", False, "@db.public.stage/source/"), + ("src/app*", None, False, "@db.public.stage/"), + ("src/app[1-5].py", None, False, "@db.public.stage/"), + ], +) +@mock.patch("snowflake.cli.api.cli_global_context.get_cli_context") +def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_path): + mock_connection = mock.Mock() + mock_connection.database = "db" + mock_connection.schema = "public" + mock_ctx_context.return_value.connection = mock_connection + + with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): + upload_path = Artefact(Path(), Path(path), dest).upload_path("stage") + + assert upload_path == expected_path + + +@pytest.mark.parametrize( + "path, dest, is_file, expected_path", + [ + ("src", None, False, Path("output") / "src.zip"), + ("src/", None, False, Path("output") / "src.zip"), + ("src", "source", False, Path("output") / "source" / "src.zip"), + ("src/app.py", None, True, Path("output") / "src" / "app.py"), + ( + "src/app.py", + "source/new_app.py", + True, + Path("output") / "source" / "new_app.py", + ), + ("src/*", "source/new_app.py", True, Path("output") / "source" / "new_app.py"), + ( + "src/dir/dir2/app.py", + None, + True, + Path("output") / "src" / "dir" / "dir2" / "app.py", + ), + ( + "src/dir/dir2/app.py", + "source/", + True, + Path("output") / "source" / "app.py", + ), + ("src/*", "source/", False, Path("output") / "source" / "src.zip"), + ("src/**/*.py", None, False, Path("output") / "src.zip"), + ("src/**/*.py", "source/", False, Path("output") / "source" / "src.zip"), + ("src/app*", None, False, Path("output") / "src.zip"), + ("src/app[1-5].py", None, False, Path("output") / "src.zip"), + ], +) +def test_artifact_post_build_path(path, dest, is_file, expected_path): + with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): + post_build_path = Artefact(Path(), Path(path), dest).post_build_path + + assert post_build_path == expected_path + + +@pytest.mark.parametrize( + "path, dest, is_file, expected_path", + [ + ("src", None, False, "@db.public.stage/src.zip"), + ("src/", None, False, "@db.public.stage/src.zip"), + ("src", "source", False, "@db.public.stage/source/src.zip"), + ("src/app.py", None, True, "@db.public.stage/src/app.py"), + ("src/app.py", "source/new_app.py", True, "@db.public.stage/source/new_app.py"), + ("src/dir/dir2/app.py", None, True, "@db.public.stage/src/dir/dir2/app.py"), + ("src/dir/dir2/app.py", "source/", True, "@db.public.stage/source/app.py"), + ("src/*", "source/", False, "@db.public.stage/source/src.zip"), + ("src/**/*.py", None, False, "@db.public.stage/src.zip"), + ("src/**/*.py", "source/", False, "@db.public.stage/source/src.zip"), + ("src/app*", None, False, "@db.public.stage/src.zip"), + ("src/app[1-5].py", None, False, "@db.public.stage/src.zip"), + ], +) +@mock.patch("snowflake.cli.api.cli_global_context.get_cli_context") +def test_artifact_import_path_from_other_directory( + mock_ctx_context, path, dest, is_file, expected_path +): + mock_connection = mock.Mock() + mock_connection.database = "db" + mock_connection.schema = "public" + mock_ctx_context.return_value.connection = mock_connection + stage = "stage" + + with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): + import_path = Artefact(Path("/tmp"), Path(path), dest).import_path(stage) + + assert import_path == expected_path + + +@pytest.mark.parametrize( + "path, dest, is_file, expected_path", + [ + ("src", None, False, "@db.public.stage/"), + ("src/", None, False, "@db.public.stage/"), + ("src", "source", False, "@db.public.stage/source/"), + ("src/app.py", None, True, "@db.public.stage/src/"), + ("src/app.py", "source/new_app.py", True, "@db.public.stage/source/"), + ("src/dir/dir2/app.py", None, True, "@db.public.stage/src/dir/dir2/"), + ("src/dir/dir2/app.py", "source/", True, "@db.public.stage/source/"), + ("src/*", "source/", False, "@db.public.stage/source/"), + ("src/**/*.py", None, False, "@db.public.stage/"), + ("src/**/*.py", "source/", False, "@db.public.stage/source/"), + ("src/app*", None, False, "@db.public.stage/"), + ("src/app[1-5].py", None, False, "@db.public.stage/"), + ], +) +@mock.patch("snowflake.cli.api.cli_global_context.get_cli_context") +def test_artifact_upload_path_from_other_directory( + mock_ctx_context, path, dest, is_file, expected_path +): + mock_connection = mock.Mock() + mock_connection.database = "db" + mock_connection.schema = "public" + mock_ctx_context.return_value.connection = mock_connection + + with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): + upload_path = Artefact(Path("/tmp"), Path(path), dest).upload_path("stage") + + assert upload_path == expected_path + + +@pytest.mark.parametrize( + "path, dest, is_file, expected_path", + [ + ("src", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ( + "src", + "source", + False, + Path.cwd().absolute() / "output" / "source" / "src.zip", + ), + ("src/app.py", None, True, Path.cwd().absolute() / "output" / "src" / "app.py"), + ( + "src/app.py", + "source/new_app.py", + True, + Path.cwd().absolute() / "output" / "source" / "new_app.py", + ), + ( + "src/dir/dir2/app.py", + None, + True, + Path.cwd().absolute() / "output" / "src" / "dir" / "dir2" / "app.py", + ), + ( + "src/dir/dir2/app.py", + "source/", + True, + Path.cwd().absolute() / "output" / "source" / "app.py", + ), + ( + "src/*", + "source/", + False, + Path.cwd().absolute() / "output" / "source" / "src.zip", + ), + ("src/**/*.py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ( + "src/**/*.py", + "source/", + False, + Path.cwd().absolute() / "output" / "source" / "src.zip", + ), + ("src/app*", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/app[1-5].py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ], +) +def test_artifact_post_build_path_from_other_directory( + path, dest, is_file, expected_path +): + with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): + post_build_path = Artefact( + Path.cwd().absolute(), Path(path), dest + ).post_build_path + + assert post_build_path == expected_path diff --git a/tests/streamlit/test_artifacts.py b/tests/streamlit/test_artifacts.py new file mode 100644 index 0000000000..4b75406d1c --- /dev/null +++ b/tests/streamlit/test_artifacts.py @@ -0,0 +1,316 @@ +import os +from pathlib import Path +from unittest import mock + +import pytest +from snowflake.cli._plugins.connection.util import UIParameter +from snowflake.connector.compat import IS_WINDOWS + + +@pytest.mark.parametrize( + "artifacts, paths", + [ + ( + "src", + [ + {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + { + "local": Path("output") / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, + ], + ), + ( + "src/", + [ + {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + { + "local": Path("output") / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, + ], + ), + ( + "src/*", + [ + {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + { + "local": Path("output") / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, + ], + ), + ("src/*.py", [{"local": Path("output") / "src" / "app.py", "stage": "/src"}]), + ( + "src/dir/dir_app.py", + [ + { + "local": Path("output") / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + } + ], + ), + ( + {"src": "src/**/*", "dest": "source/"}, + [ + {"local": Path("output") / "source" / "app.py", "stage": "/source"}, + {"local": Path("output") / "source" / "dir_app.py", "stage": "/source"}, + { + "local": Path("output") / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, + ], + ), + ( + {"src": "src", "dest": "source/"}, + [ + { + "local": Path("output") / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "stage": "/source/src/dir", + }, + ], + ), + ( + {"src": "src/", "dest": "source/"}, + [ + { + "local": Path("output") / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "stage": "/source/src/dir", + }, + ], + ), + ( + {"src": "src/*", "dest": "source/"}, + [ + {"local": Path("output") / "source" / "app.py", "stage": "/source"}, + { + "local": Path("output") / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, + ], + ), + ( + {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, + [ + { + "local": Path("output") / "source" / "dir" / "apps" / "dir_app.py", + "stage": "/source/dir/apps", + } + ], + ), + ], +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put") +@mock.patch( + "snowflake.cli._plugins.connection.util.get_ui_parameters", + return_value={UIParameter.NA_ENABLE_REGIONLESS_REDIRECT: "false"}, +) +def test_deploy_with_artifacts( + mock_param, + mock_sm_put, + mock_conn, + mock_cursor, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + artifacts, + paths, +): + ctx = mock_ctx( + mock_cursor( + rows=[ + {"SYSTEM$GET_SNOWSIGHT_HOST()": "https://snowsight.domain"}, + {"CURRENT_ACCOUNT_NAME()": "my_account"}, + ], + columns=["SYSTEM$GET_SNOWSIGHT_HOST()"], + ) + ) + mock_conn.return_value = ctx + + streamlit_files = [ + "streamlit_app.py", + "pages/my_page.py", + "environment.yml", + ] + + with project_directory("glob_patterns") as tmp: + alter_snowflake_yml( + tmp / "snowflake.yml", + "entities.my_streamlit.artifacts", + streamlit_files + [artifacts], + ) + + result = runner.invoke( + [ + "streamlit", + "deploy", + "--replace", + ] + ) + assert result.exit_code == 0, result.output + + put_calls = _extract_put_calls(mock_sm_put) + # Windows needs absolute paths. + if IS_WINDOWS: + tmp_path = tmp.absolute() + else: + tmp_path = tmp.resolve() + for path in paths: + assert { + "local_path": tmp_path / path["local"], + "stage_path": "@MockDatabase.MockSchema.streamlit/test_streamlit_deploy_snowcli" + + path["stage"], + } in put_calls + + +@pytest.mark.parametrize( + "artifacts, paths", + [ + ( + "src", + [ + {"local": "output/src/app.py", "stage": "/src"}, + {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + ], + ), + ( + "src/", + [ + {"local": "output/src/app.py", "stage": "/src"}, + {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + ], + ), + ( + "src/*", + [ + {"local": "output/src/app.py", "stage": "/src"}, + {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + ], + ), + ("src/*.py", [{"local": "output/src/app.py", "stage": "/src"}]), + ( + "src/dir/dir_app.py", + [{"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}], + ), + ( + {"src": "src/**/*", "dest": "source/"}, + [ + {"local": "output/source/app.py", "stage": "/source"}, + {"local": "output/source/dir_app.py", "stage": "/source"}, + {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + ], + ), + ( + {"src": "src", "dest": "source/"}, + [ + {"local": "output/source/src/app.py", "stage": "/source/src"}, + { + "local": "output/source/src/dir/dir_app.py", + "stage": "/source/src/dir", + }, + ], + ), + ( + {"src": "src/", "dest": "source/"}, + [ + {"local": "output/source/src/app.py", "stage": "/source/src"}, + { + "local": "output/source/src/dir/dir_app.py", + "stage": "/source/src/dir", + }, + ], + ), + ( + {"src": "src/*", "dest": "source/"}, + [ + {"local": "output/source/app.py", "stage": "/source"}, + {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + ], + ), + ( + {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, + [ + { + "local": "output/source/dir/apps/dir_app.py", + "stage": "/source/dir/apps", + } + ], + ), + ], +) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put") +@mock.patch( + "snowflake.cli._plugins.connection.util.get_ui_parameters", + return_value={UIParameter.NA_ENABLE_REGIONLESS_REDIRECT: "false"}, +) +def test_deploy_with_artifacts_from_other_directory( + mock_param, + mock_sm_put, + mock_conn, + mock_cursor, + runner, + mock_ctx, + project_directory, + alter_snowflake_yml, + artifacts, + paths, +): + ctx = mock_ctx( + mock_cursor( + rows=[ + {"SYSTEM$GET_SNOWSIGHT_HOST()": "https://snowsight.domain"}, + {"REGIONLESS": "false"}, + {"CURRENT_ACCOUNT_NAME()": "https://snowsight.domain"}, + ], + columns=["SYSTEM$GET_SNOWSIGHT_HOST()"], + ) + ) + mock_conn.return_value = ctx + + streamlit_files = [ + "streamlit_app.py", + "pages/my_page.py", + "environment.yml", + ] + + with project_directory("glob_patterns") as tmp: + os.chdir(Path(os.getcwd()).parent) + alter_snowflake_yml( + tmp / "snowflake.yml", + "entities.my_streamlit.artifacts", + streamlit_files + [artifacts], + ) + + result = runner.invoke(["streamlit", "deploy", "-p", tmp, "--replace"]) + assert result.exit_code == 0, result.output + + put_calls = _extract_put_calls(mock_sm_put) + for path in paths: + assert { + "local_path": tmp / path["local"], + "stage_path": "@MockDatabase.MockSchema.streamlit/test_streamlit_deploy_snowcli" + + path["stage"], + } in put_calls + + +def _extract_put_calls(mock_sm_put): + # Extract the put calls from the mock for better visibility in test logs + return [ + { + "local_path": call.kwargs.get("local_path"), + "stage_path": call.kwargs.get("stage_path"), + } + for call in mock_sm_put.mock_calls + if call.kwargs.get("local_path") + ] diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index e533a66d39..cd39056335 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -55,9 +55,9 @@ def test_describe_streamlit(mock_connector, runner, mock_ctx): ] -def _put_query(source: str, dest: str): +def _put_query(project_root: Path, source: str, dest: str): return dedent( - f"put file://{Path(source)} {dest} auto_compress=false parallel=4 overwrite=True" + f"put file://{project_root.resolve() / 'output' / source} {dest} auto_compress=false parallel=4 overwrite=True" ) @@ -91,16 +91,18 @@ def test_deploy_only_streamlit_file( mock_connector.return_value = ctx mock_get_account.return_value = "my_account" - with project_directory("example_streamlit") as pdir: - (pdir / "environment.yml").unlink() - shutil.rmtree(pdir / "pages") + with project_directory("example_streamlit") as tmp_dir: + (tmp_dir / "environment.yml").unlink() + shutil.rmtree(tmp_dir / "pages") result = runner.invoke(["streamlit", "deploy"]) assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", _put_query( - "streamlit_app.py", "@MockDatabase.MockSchema.streamlit/test_streamlit" + tmp_dir, + "streamlit_app.py", + "@MockDatabase.MockSchema.streamlit/test_streamlit", ), dedent( f""" @@ -146,16 +148,18 @@ def test_deploy_only_streamlit_file_no_stage( mock_connector.return_value = ctx mock_get_account.return_value = "my_account" - with project_directory("example_streamlit_no_stage") as pdir: - (pdir / "environment.yml").unlink() - shutil.rmtree(pdir / "pages") + with project_directory("example_streamlit_no_stage") as tmp_dir: + (tmp_dir / "environment.yml").unlink() + shutil.rmtree(tmp_dir / "pages") result = runner.invoke(["streamlit", "deploy"]) assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", _put_query( - "streamlit_app.py", "@MockDatabase.MockSchema.streamlit/test_streamlit" + tmp_dir, + "streamlit_app.py", + "@MockDatabase.MockSchema.streamlit/test_streamlit", ), dedent( f""" @@ -200,18 +204,22 @@ def test_deploy_with_empty_pages( mock_connector.return_value = ctx mock_get_account.return_value = "my_account" - with project_directory("streamlit_empty_pages") as directory: - (directory / "pages").mkdir(parents=True, exist_ok=True) + with project_directory("streamlit_empty_pages") as tmp_dir: + (tmp_dir / "pages").mkdir(parents=True, exist_ok=True) result = runner.invoke(["streamlit", "deploy"]) assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", _put_query( - "streamlit_app.py", "@MockDatabase.MockSchema.streamlit/test_streamlit" + tmp_dir, + "streamlit_app.py", + "@MockDatabase.MockSchema.streamlit/test_streamlit", ), _put_query( - "environment.yml", "@MockDatabase.MockSchema.streamlit/test_streamlit" + tmp_dir, + "environment.yml", + "@MockDatabase.MockSchema.streamlit/test_streamlit", ), dedent( f""" @@ -223,7 +231,6 @@ def test_deploy_with_empty_pages( ).strip(), "select system$get_snowsight_host()", ] - assert "Skipping empty directory: pages" in result.output @mock.patch("snowflake.cli._plugins.connection.util.get_account") @@ -256,16 +263,18 @@ def test_deploy_only_streamlit_file_replace( mock_connector.return_value = ctx mock_get_account.return_value = "my_account" - with project_directory("example_streamlit") as pdir: - (pdir / "environment.yml").unlink() - shutil.rmtree(pdir / "pages") + with project_directory("example_streamlit") as tmp_dir: + (tmp_dir / "environment.yml").unlink() + shutil.rmtree(tmp_dir / "pages") result = runner.invoke(["streamlit", "deploy", "--replace"]) assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", _put_query( - "streamlit_app.py", "@MockDatabase.MockSchema.streamlit/test_streamlit" + tmp_dir, + "streamlit_app.py", + "@MockDatabase.MockSchema.streamlit/test_streamlit", ), dedent( f""" @@ -281,23 +290,6 @@ def test_deploy_only_streamlit_file_replace( mock_typer.launch.assert_not_called() -def test_artifacts_must_exists( - runner, mock_ctx, project_directory, alter_snowflake_yml, snapshot -): - with project_directory("example_streamlit_v2") as pdir: - alter_snowflake_yml( - pdir / "snowflake.yml", - parameter_path="entities.my_streamlit.artifacts.1", - value="foo_bar.py", - ) - - result = runner.invoke( - ["streamlit", "deploy"], - ) - assert result.exit_code == 1 - assert result.output == snapshot - - @pytest.mark.parametrize("project_name", ["example_streamlit_v2", "example_streamlit"]) @mock.patch("snowflake.cli._plugins.streamlit.commands.typer") @mock.patch("snowflake.connector.connect") @@ -364,11 +356,11 @@ def test_deploy_streamlit_and_environment_files( ) mock_connector.return_value = ctx - with project_directory(project_name) as pdir: - shutil.rmtree(pdir / "pages") + with project_directory(project_name) as tmp_dir: + shutil.rmtree(tmp_dir / "pages") if project_name == "example_streamlit_v2": alter_snowflake_yml( - pdir / "snowflake.yml", + tmp_dir / "snowflake.yml", parameter_path="entities.test_streamlit.artifacts", value=["streamlit_app.py", "environment.yml"], ) @@ -379,8 +371,8 @@ def test_deploy_streamlit_and_environment_files( assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", root_path), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", root_path), dedent( f""" CREATE STREAMLIT IDENTIFIER('MockDatabase.MockSchema.{STREAMLIT_NAME}') @@ -423,11 +415,11 @@ def test_deploy_streamlit_and_pages_files( ) mock_connector.return_value = ctx - with project_directory(project_name) as pdir: - (pdir / "environment.yml").unlink() + with project_directory(project_name) as tmp_dir: + (tmp_dir / "environment.yml").unlink() if project_name == "example_streamlit_v2": alter_snowflake_yml( - pdir / "snowflake.yml", + tmp_dir / "snowflake.yml", parameter_path="entities.test_streamlit.artifacts", value=["streamlit_app.py", "pages/"], ) @@ -437,8 +429,8 @@ def test_deploy_streamlit_and_pages_files( assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", - _put_query("streamlit_app.py", root_path), - _put_query("pages/*", f"{root_path}/pages"), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), dedent( f""" CREATE STREAMLIT IDENTIFIER('MockDatabase.MockSchema.{STREAMLIT_NAME}') @@ -482,18 +474,18 @@ def test_deploy_all_streamlit_files( ) mock_connector.return_value = ctx - with project_directory(project_name): + with project_directory(project_name) as tmp_dir: result = runner.invoke(["streamlit", "deploy"]) root_path = f"@MockDatabase.MockSchema.streamlit/{STREAMLIT_NAME}" assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", root_path), - _put_query("pages/*", f"{root_path}/pages"), - _put_query("utils/utils.py", f"{root_path}/utils"), - _put_query("extra_file.py", root_path), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", root_path), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), + _put_query(tmp_dir, "utils/utils.py", f"{root_path}/utils"), + _put_query(tmp_dir, "extra_file.py", root_path), dedent( f""" CREATE STREAMLIT IDENTIFIER('MockDatabase.MockSchema.{STREAMLIT_NAME}') @@ -555,16 +547,16 @@ def test_deploy_put_files_on_stage( with project_directory( project_name, merge_project_definition=merge_definition, - ): + ) as tmp_dir: result = runner.invoke(["streamlit", "deploy"]) root_path = f"@MockDatabase.MockSchema.streamlit_stage/{STREAMLIT_NAME}" assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit_stage')", - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", root_path), - _put_query("pages/*", f"{root_path}/pages"), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", root_path), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), dedent( f""" CREATE STREAMLIT IDENTIFIER('MockDatabase.MockSchema.{STREAMLIT_NAME}') @@ -609,16 +601,18 @@ def test_deploy_all_streamlit_files_not_defaults( ) mock_connector.return_value = ctx - with project_directory(project_name): + with project_directory(project_name) as tmp_dir: result = runner.invoke(["streamlit", "deploy"]) root_path = f"@MockDatabase.MockSchema.streamlit_stage/{STREAMLIT_NAME}" assert result.exit_code == 0, result.output assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit_stage')", - _put_query("main.py", root_path), - _put_query("streamlit_environment.yml", root_path), - _put_query("streamlit_pages/*", f"{root_path}/streamlit_pages"), + _put_query(tmp_dir, "main.py", root_path), + _put_query(tmp_dir, "streamlit_environment.yml", root_path), + _put_query( + tmp_dir, "streamlit_pages/first_page.py", f"{root_path}/streamlit_pages" + ), dedent( f""" CREATE STREAMLIT IDENTIFIER('MockDatabase.MockSchema.{STREAMLIT_NAME}') @@ -675,10 +669,10 @@ def test_deploy_streamlit_main_and_pages_files_experimental( return_value=enable_streamlit_no_checkouts, ), ): - with project_directory(project_name) as pdir: + with project_directory(project_name) as tmp_dir: if project_name == "example_streamlit_v2": alter_snowflake_yml( - pdir / "snowflake.yml", + tmp_dir / "snowflake.yml", parameter_path="entities.test_streamlit.artifacts", value=["streamlit_app.py", "environment.yml", "pages"], ) @@ -711,9 +705,9 @@ def test_deploy_streamlit_main_and_pages_files_experimental( """ ).strip(), post_create_command, - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", f"{root_path}"), - _put_query("pages/*", f"{root_path}/pages"), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", f"{root_path}"), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), "select system$get_snowsight_host()", "select current_account_name()", ] @@ -771,19 +765,17 @@ def test_deploy_streamlit_main_and_pages_files_experimental_double_deploy( ) ctx.queries = [] - with project_directory(project_name) as pdir: + with project_directory(project_name) as tmp_dir: if project_name == "example_streamlit_v2": alter_snowflake_yml( - pdir / "snowflake.yml", + tmp_dir / "snowflake.yml", parameter_path="entities.test_streamlit.artifacts", value=["streamlit_app.py", "environment.yml", "pages"], ) result2 = runner.invoke(["streamlit", "deploy", "--experimental"]) - assert result2.exit_code == 0, result2.output - root_path = f"@streamlit/MockDatabase.MockSchema.{STREAMLIT_NAME}/default_checkout" - + assert result2.exit_code == 0, result2.output # Same as normal, except no ALTER query assert ctx.get_queries() == [ dedent( @@ -794,9 +786,9 @@ def test_deploy_streamlit_main_and_pages_files_experimental_double_deploy( TITLE = 'My Fancy Streamlit' """ ).strip(), - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", f"{root_path}"), - _put_query("pages/*", f"{root_path}/pages"), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", f"{root_path}"), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), "select system$get_snowsight_host()", "select current_account_name()", ] @@ -838,7 +830,7 @@ def test_deploy_streamlit_main_and_pages_files_experimental_no_stage( "snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_STREAMLIT_VERSIONED_STAGE.is_enabled", return_value=enable_streamlit_versioned_stage, ): - with project_directory(project_name): + with project_directory(project_name) as tmp_dir: result = runner.invoke(["streamlit", "deploy", "--experimental"]) @@ -863,9 +855,9 @@ def test_deploy_streamlit_main_and_pages_files_experimental_no_stage( """ ).strip(), post_create_command, - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", f"{root_path}"), - _put_query("pages/*", f"{root_path}/pages"), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", f"{root_path}"), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), f"select system$get_snowsight_host()", f"select current_account_name()", ] @@ -899,10 +891,10 @@ def test_deploy_streamlit_main_and_pages_files_experimental_replace( ) mock_connector.return_value = ctx - with project_directory(project_name) as pdir: + with project_directory(project_name) as tmp_dir: if project_name == "example_streamlit_v2": alter_snowflake_yml( - pdir / "snowflake.yml", + tmp_dir / "snowflake.yml", parameter_path="entities.test_streamlit.artifacts", value=["streamlit_app.py", "environment.yml", "pages/"], ) @@ -920,9 +912,9 @@ def test_deploy_streamlit_main_and_pages_files_experimental_replace( """ ).strip(), f"ALTER streamlit MockDatabase.MockSchema.{STREAMLIT_NAME} CHECKOUT", - _put_query("streamlit_app.py", root_path), - _put_query("environment.yml", f"{root_path}"), - _put_query("pages/*", f"{root_path}/pages"), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "environment.yml", f"{root_path}"), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), f"select system$get_snowsight_host()", f"select current_account_name()", ] @@ -1084,7 +1076,7 @@ def test_deploy_streamlit_with_comment_v2( ) mock_connector.return_value = ctx - with project_directory("example_streamlit_with_comment_v2"): + with project_directory("example_streamlit_with_comment_v2") as tmp_dir: result = runner.invoke(["streamlit", "deploy", "--replace"]) root_path = f"@MockDatabase.MockSchema.streamlit/test_streamlit_deploy_snowcli" @@ -1092,9 +1084,9 @@ def test_deploy_streamlit_with_comment_v2( assert ctx.get_queries() == [ f"describe streamlit IDENTIFIER('MockDatabase.MockSchema.test_streamlit_deploy_snowcli')", "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.streamlit')", - _put_query("streamlit_app.py", root_path), - _put_query("pages/*", f"{root_path}/pages"), - _put_query("environment.yml", root_path), + _put_query(tmp_dir, "streamlit_app.py", root_path), + _put_query(tmp_dir, "pages/my_page.py", f"{root_path}/pages"), + _put_query(tmp_dir, "environment.yml", root_path), dedent( f""" CREATE OR REPLACE STREAMLIT IDENTIFIER('MockDatabase.MockSchema.test_streamlit_deploy_snowcli') diff --git a/tests/streamlit/test_streamlit_manager.py b/tests/streamlit/test_streamlit_manager.py index 4f04fe854c..6197d6c58b 100644 --- a/tests/streamlit/test_streamlit_manager.py +++ b/tests/streamlit/test_streamlit_manager.py @@ -7,6 +7,9 @@ from snowflake.cli._plugins.streamlit.streamlit_entity_model import ( StreamlitEntityModel, ) +from snowflake.cli._plugins.streamlit.streamlit_project_paths import ( + StreamlitProjectPaths, +) from snowflake.cli.api.identifiers import FQN mock_streamlit_exists = mock.patch( @@ -22,7 +25,7 @@ def test_deploy_streamlit(mock_execute_query, _, mock_stage_manager, temp_dir): mock_stage_manager().get_standard_stage_prefix.return_value = "stage_root" - main_file = Path(temp_dir) / "main.py" + main_file = Path("main.py") main_file.touch() st = StreamlitEntityModel( @@ -33,11 +36,13 @@ def test_deploy_streamlit(mock_execute_query, _, mock_stage_manager, temp_dir): main_file=str(main_file), imports=["@stage/foo.py", "@stage/bar.py"], # Possibly can be PathMapping - artifacts=[main_file], + artifacts=[str(main_file)], ) + streamlit_project_paths = StreamlitProjectPaths(Path().absolute()) + StreamlitManager(MagicMock(database="DB", schema="SH")).deploy( - streamlit=st, replace=False + streamlit=st, streamlit_project_paths=streamlit_project_paths, replace=False ) mock_execute_query.assert_called_once_with( @@ -62,7 +67,7 @@ def test_deploy_streamlit_with_api_integrations( ): mock_stage_manager().get_standard_stage_prefix.return_value = "stage_root" - main_file = Path(temp_dir) / "main.py" + main_file = Path("main.py") main_file.touch() st = StreamlitEntityModel( @@ -72,13 +77,15 @@ def test_deploy_streamlit_with_api_integrations( query_warehouse="My_WH", main_file=str(main_file), # Possibly can be PathMapping - artifacts=[main_file], + artifacts=[str(main_file)], external_access_integrations=["MY_INTERGATION", "OTHER"], secrets={"my_secret": "SecretOfTheSecrets", "other": "other_secret"}, ) + streamlit_project_paths = StreamlitProjectPaths(Path().absolute()) + StreamlitManager(MagicMock(database="DB", schema="SH")).deploy( - streamlit=st, replace=False + streamlit=st, streamlit_project_paths=streamlit_project_paths, replace=False ) mock_execute_query.assert_called_once_with( @@ -104,7 +111,7 @@ def test_deploy_streamlit_with_comment( ): mock_stage_manager().get_standard_stage_prefix.return_value = "stage_root" - main_file = Path(temp_dir) / "main.py" + main_file = Path("main.py") main_file.touch() st = StreamlitEntityModel( @@ -113,12 +120,14 @@ def test_deploy_streamlit_with_comment( title="MyStreamlit", query_warehouse="My_WH", main_file=str(main_file), - artifacts=[main_file], + artifacts=[str(main_file)], comment="This is a test comment", ) + streamlit_project_paths = StreamlitProjectPaths(Path().absolute()) + StreamlitManager(MagicMock(database="DB", schema="SH")).deploy( - streamlit=st, replace=False + streamlit=st, streamlit_project_paths=streamlit_project_paths, replace=False ) mock_execute_query.assert_called_once_with( diff --git a/tests/test_data/projects/glob_patterns/environment.yml b/tests/test_data/projects/glob_patterns/environment.yml new file mode 100644 index 0000000000..ac8feac3e8 --- /dev/null +++ b/tests/test_data/projects/glob_patterns/environment.yml @@ -0,0 +1,5 @@ +name: sf_env +channels: + - snowflake +dependencies: + - pandas diff --git a/tests/test_data/projects/glob_patterns/main.py b/tests/test_data/projects/glob_patterns/main.py new file mode 100644 index 0000000000..52c7b0751f --- /dev/null +++ b/tests/test_data/projects/glob_patterns/main.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import sys + +from procedures import hello_procedure +from snowflake.snowpark import Session + +# For local debugging. Be aware you may need to type-convert arguments if +# you add input parameters +if __name__ == "__main__": + from snowflake.cli.api.config import cli_config + + session = Session.builder.configs(cli_config.get_connection_dict("dev")).create() + if len(sys.argv) > 1: + print(hello_procedure(session, *sys.argv[1:])) # type: ignore + else: + print(hello_procedure(session)) # type: ignore + session.close() diff --git a/tests/test_data/projects/glob_patterns/pages/my_page.py b/tests/test_data/projects/glob_patterns/pages/my_page.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_data/projects/glob_patterns/snowflake.yml b/tests/test_data/projects/glob_patterns/snowflake.yml new file mode 100644 index 0000000000..38c39e7566 --- /dev/null +++ b/tests/test_data/projects/glob_patterns/snowflake.yml @@ -0,0 +1,23 @@ +definition_version: 2 +entities: + hello_procedure: + artifacts: + - # set in test + handler: hello + identifier: + name: hello_procedure + returns: string + signature: + - name: "name" + type: "string" + stage: dev_deployment + type: procedure + my_streamlit: + type: "streamlit" + identifier: test_streamlit_deploy_snowcli + title: "My Fancy Streamlit" + stage: streamlit + query_warehouse: xsmall + main_file: streamlit_app.py + artifacts: + - # set in test diff --git a/tests/test_data/projects/glob_patterns/src/app.py b/tests/test_data/projects/glob_patterns/src/app.py new file mode 100644 index 0000000000..6dac2047d2 --- /dev/null +++ b/tests/test_data/projects/glob_patterns/src/app.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from dir.dir_app import print_hello +from snowflake.snowpark import Session + + +def hello_procedure(session: Session, name: str) -> str: + return print_hello(name) + + +def hello_function(name: str) -> str: + return print_hello(name) diff --git a/tests/test_data/projects/glob_patterns/src/dir/dir_app.py b/tests/test_data/projects/glob_patterns/src/dir/dir_app.py new file mode 100644 index 0000000000..055739db8d --- /dev/null +++ b/tests/test_data/projects/glob_patterns/src/dir/dir_app.py @@ -0,0 +1,2 @@ +def print_hello(name: str): + print(f"Hello, {name}!") diff --git a/tests/test_data/projects/glob_patterns/streamlit_app.py b/tests/test_data/projects/glob_patterns/streamlit_app.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_data/projects/glob_patterns_zip/commons/helpers.py b/tests/test_data/projects/glob_patterns_zip/commons/helpers.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_data/projects/glob_patterns_zip/environment.yml b/tests/test_data/projects/glob_patterns_zip/environment.yml new file mode 100644 index 0000000000..ac8feac3e8 --- /dev/null +++ b/tests/test_data/projects/glob_patterns_zip/environment.yml @@ -0,0 +1,5 @@ +name: sf_env +channels: + - snowflake +dependencies: + - pandas diff --git a/tests/test_data/projects/glob_patterns_zip/main.py b/tests/test_data/projects/glob_patterns_zip/main.py new file mode 100644 index 0000000000..52c7b0751f --- /dev/null +++ b/tests/test_data/projects/glob_patterns_zip/main.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import sys + +from procedures import hello_procedure +from snowflake.snowpark import Session + +# For local debugging. Be aware you may need to type-convert arguments if +# you add input parameters +if __name__ == "__main__": + from snowflake.cli.api.config import cli_config + + session = Session.builder.configs(cli_config.get_connection_dict("dev")).create() + if len(sys.argv) > 1: + print(hello_procedure(session, *sys.argv[1:])) # type: ignore + else: + print(hello_procedure(session)) # type: ignore + session.close() diff --git a/tests/test_data/projects/glob_patterns_zip/snowflake.yml b/tests/test_data/projects/glob_patterns_zip/snowflake.yml new file mode 100644 index 0000000000..38c39e7566 --- /dev/null +++ b/tests/test_data/projects/glob_patterns_zip/snowflake.yml @@ -0,0 +1,23 @@ +definition_version: 2 +entities: + hello_procedure: + artifacts: + - # set in test + handler: hello + identifier: + name: hello_procedure + returns: string + signature: + - name: "name" + type: "string" + stage: dev_deployment + type: procedure + my_streamlit: + type: "streamlit" + identifier: test_streamlit_deploy_snowcli + title: "My Fancy Streamlit" + stage: streamlit + query_warehouse: xsmall + main_file: streamlit_app.py + artifacts: + - # set in test diff --git a/tests/test_data/projects/glob_patterns_zip/src/app.py b/tests/test_data/projects/glob_patterns_zip/src/app.py new file mode 100644 index 0000000000..6dac2047d2 --- /dev/null +++ b/tests/test_data/projects/glob_patterns_zip/src/app.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from dir.dir_app import print_hello +from snowflake.snowpark import Session + + +def hello_procedure(session: Session, name: str) -> str: + return print_hello(name) + + +def hello_function(name: str) -> str: + return print_hello(name) diff --git a/tests/test_data/projects/glob_patterns_zip/src/dir/dir_app.py b/tests/test_data/projects/glob_patterns_zip/src/dir/dir_app.py new file mode 100644 index 0000000000..055739db8d --- /dev/null +++ b/tests/test_data/projects/glob_patterns_zip/src/dir/dir_app.py @@ -0,0 +1,2 @@ +def print_hello(name: str): + print(f"Hello, {name}!") diff --git a/tests/test_data/projects/glob_patterns_zip/streamlit_app.py b/tests/test_data/projects/glob_patterns_zip/streamlit_app.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/testing_utils/fixtures.py b/tests/testing_utils/fixtures.py index ee731735b1..457acec1c6 100644 --- a/tests/testing_utils/fixtures.py +++ b/tests/testing_utils/fixtures.py @@ -466,3 +466,12 @@ def mock_procedure_description(mock_cursor): "installed_packages", ], ) + + +@pytest.fixture +def enable_snowpark_glob_support_feature_flag(): + with mock.patch( + f"snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled", + return_value=True, + ): + yield diff --git a/tests_integration/conftest.py b/tests_integration/conftest.py index 02d6fba10c..6be62f1ce5 100644 --- a/tests_integration/conftest.py +++ b/tests_integration/conftest.py @@ -24,6 +24,7 @@ from json import JSONDecodeError from pathlib import Path from typing import Any, Dict, List, Optional +from unittest import mock from uuid import uuid4 import pytest @@ -258,3 +259,12 @@ def resource_suffix(request): # To generate a suffix that isn't too long or complex, we use originalname, which is the # "bare" test function name, without filename, class name, or parameterization variables return f"_{uuid4().hex}_{request.node.originalname}" + + +@pytest.fixture +def enable_snowpark_glob_support_feature_flag(): + with mock.patch( + f"snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled", + return_value=True, + ): + yield diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/app_1/a.py b/tests_integration/test_data/projects/snowpark_glob_patterns/app_1/a.py new file mode 100644 index 0000000000..f92e382069 --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/app_1/a.py @@ -0,0 +1,12 @@ +from __future__ import annotations +from snowflake.snowpark import Session +from b import test_procedure + + +# test import +import syrupy + + +def hello_procedure(session: Session, name: str) -> str: + + return f"Hello {name}" + test_procedure(session) diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/app_1/b.py b/tests_integration/test_data/projects/snowpark_glob_patterns/app_1/b.py new file mode 100644 index 0000000000..bef124997f --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/app_1/b.py @@ -0,0 +1,10 @@ +from __future__ import annotations +from snowflake.snowpark import Session + + +# test import +import syrupy + + +def test_procedure(session: Session) -> str: + return "Test procedure" diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/app_2/c.py b/tests_integration/test_data/projects/snowpark_glob_patterns/app_2/c.py new file mode 100644 index 0000000000..f92e382069 --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/app_2/c.py @@ -0,0 +1,12 @@ +from __future__ import annotations +from snowflake.snowpark import Session +from b import test_procedure + + +# test import +import syrupy + + +def hello_procedure(session: Session, name: str) -> str: + + return f"Hello {name}" + test_procedure(session) diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/app_2/d.py b/tests_integration/test_data/projects/snowpark_glob_patterns/app_2/d.py new file mode 100644 index 0000000000..bef124997f --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/app_2/d.py @@ -0,0 +1,10 @@ +from __future__ import annotations +from snowflake.snowpark import Session + + +# test import +import syrupy + + +def test_procedure(session: Session) -> str: + return "Test procedure" diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/e.py b/tests_integration/test_data/projects/snowpark_glob_patterns/e.py new file mode 100644 index 0000000000..3ab4a6d6cc --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/e.py @@ -0,0 +1,9 @@ +from __future__ import annotations + + +# test import +import syrupy + + +def hello_function(name: str) -> str: + return f"Hello {name}!" diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/requirements.txt b/tests_integration/test_data/projects/snowpark_glob_patterns/requirements.txt new file mode 100644 index 0000000000..18af07a40d --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/requirements.txt @@ -0,0 +1 @@ +snowflake-snowpark-python syrupy \ No newline at end of file diff --git a/tests_integration/test_data/projects/snowpark_glob_patterns/snowflake.yml b/tests_integration/test_data/projects/snowpark_glob_patterns/snowflake.yml new file mode 100644 index 0000000000..1275f19916 --- /dev/null +++ b/tests_integration/test_data/projects/snowpark_glob_patterns/snowflake.yml @@ -0,0 +1,43 @@ +definition_version: 2 + +mixins: + snowpark_shared: + stage: "dev_deployment" + +entities: + hello_procedure: + type: "procedure" + stage: "stage_a" + identifier: + name: "hello_procedure" + handler: "a.hello_procedure" + signature: + - name: "name" + type: "string" + returns: string + artifacts: + - "app_1/*" + + test: + type: "procedure" + handler: "d.test_procedure" + signature: "" + returns: string + artifacts: + - "app_2/*" + meta: + use_mixins: + - "snowpark_shared" + + hello_function: + type: "function" + handler: "e.hello_function" + signature: + - name: "name" + type: "string" + returns: string + artifacts: + - "e.py" + meta: + use_mixins: + - "snowpark_shared" diff --git a/tests_integration/test_snowpark.py b/tests_integration/test_snowpark.py index 7701581f74..c55f33ee90 100644 --- a/tests_integration/test_snowpark.py +++ b/tests_integration/test_snowpark.py @@ -37,6 +37,313 @@ @pytest.mark.integration def test_snowpark_flow( + _test_steps, + project_directory, + alter_snowflake_yml, + test_database, + enable_snowpark_glob_support_feature_flag, +): + database = test_database.upper() + with project_directory("snowpark") as tmp_dir: + _test_steps.snowpark_build_should_zip_files( + additional_files=[ + Path("output"), + Path("output") / "my_snowpark_project", + Path("output") / "my_snowpark_project" / "app.zip", + ] + ) + + _test_steps.snowpark_deploy_should_finish_successfully_and_return( + [ + { + "object": f"{database}.PUBLIC.hello_procedure(name string)", + "status": "created", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.test()", + "status": "created", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.hello_function(name string)", + "status": "created", + "type": "function", + }, + ] + ) + + _test_steps.assert_those_procedures_are_in_snowflake( + "hello_procedure(VARCHAR) RETURN VARCHAR" + ) + _test_steps.assert_those_functions_are_in_snowflake( + "hello_function(VARCHAR) RETURN VARCHAR" + ) + + expected_files = [ + f"{STAGE_NAME}/my_snowpark_project/app.zip", + f"{STAGE_NAME}/dependencies.zip", + ] + _test_steps.assert_that_only_these_files_are_staged_in_test_db( + *expected_files, stage_name=STAGE_NAME + ) + + # Listing procedures or functions shows created objects + _test_steps.object_show_includes_given_identifiers( + object_type="procedure", + identifier=("hello_procedure", "(VARCHAR) RETURN VARCHAR"), + ) + _test_steps.object_show_includes_given_identifiers( + object_type="function", + identifier=("hello_function", "(VARCHAR) RETURN VARCHAR"), + ) + + # Created objects can be described + _test_steps.object_describe_should_return_entity_description( + object_type="procedure", + identifier="hello_procedure(VARCHAR)", + signature="(NAME VARCHAR)", + returns=RETURN_TYPE, + ) + + _test_steps.object_describe_should_return_entity_description( + object_type="function", + identifier="hello_function(VARCHAR)", + signature="(NAME VARCHAR)", + returns=RETURN_TYPE, + ) + + # Grants are given correctly + + _test_steps.set_grants_on_selected_object( + object_type="procedure", + object_name="hello_procedure(VARCHAR)", + privillege="USAGE", + role="test_role", + ) + + _test_steps.set_grants_on_selected_object( + object_type="function", + object_name="hello_function(VARCHAR)", + privillege="USAGE", + role="test_role", + ) + + _test_steps.assert_that_object_has_expected_grant( + object_type="procedure", + object_name="hello_procedure(VARCHAR)", + expected_privillege="USAGE", + expected_role="test_role", + ) + + _test_steps.assert_that_object_has_expected_grant( + object_type="function", + object_name="hello_function(VARCHAR)", + expected_privillege="USAGE", + expected_role="test_role", + ) + + # Created objects can be executed + _test_steps.snowpark_execute_should_return_expected_value( + object_type="procedure", + identifier="hello_procedure('foo')", + expected_value="Hello foo", + ) + + _test_steps.snowpark_execute_should_return_expected_value( + object_type="function", + identifier="hello_function('foo')", + expected_value="Hello foo!", + ) + + # Subsequent deploy of same object should fail + _test_steps.snowpark_deploy_should_return_error_with_message_contains( + "Following objects already exists" + ) + + # Apply changes to project objects + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.procedures.0.returns", + value="variant", + ) + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.functions.0.returns", + value="variant", + ) + + # Now we deploy with replace flag, it should update existing objects + _test_steps.snowpark_deploy_should_finish_successfully_and_return( + additional_arguments=["--replace"], + expected_result=[ + { + "object": f"{database}.PUBLIC.hello_procedure(name string)", + "status": "definition updated", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.test()", + "status": "packages updated", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.hello_function(name string)", + "status": "definition updated", + "type": "function", + }, + ], + ) + + # Apply another changes to project objects + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.procedures.0.execute_as_caller", + value="true", + ) + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.functions.0.runtime", + value="3.11", + ) + + # Another deploy with replace flag, it should update existing objects + _test_steps.snowpark_deploy_should_finish_successfully_and_return( + additional_arguments=["--replace"], + expected_result=[ + { + "object": f"{database}.PUBLIC.hello_procedure(name string)", + "status": "definition updated", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.test()", + "status": "packages updated", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.hello_function(name string)", + "status": "definition updated", + "type": "function", + }, + ], + ) + + # Check if objects were updated + _test_steps.assert_those_procedures_are_in_snowflake( + "hello_procedure(VARCHAR) RETURN VARIANT" + ) + _test_steps.assert_those_functions_are_in_snowflake( + "hello_function(VARCHAR) RETURN VARIANT" + ) + + _test_steps.assert_that_only_these_files_are_staged_in_test_db( + *expected_files, stage_name=STAGE_NAME + ) + + # Listing procedures or functions shows updated objects + _test_steps.object_show_includes_given_identifiers( + object_type="procedure", + identifier=("hello_procedure", "(VARCHAR) RETURN VARIANT"), + ) + _test_steps.object_show_includes_given_identifiers( + object_type="function", + identifier=("hello_function", "(VARCHAR) RETURN VARIANT"), + ) + + # Updated objects can be executed + _test_steps.snowpark_execute_should_return_expected_value( + object_type="procedure", + identifier="hello_procedure('foo')", + expected_value='"Hello foo"', + ) + + _test_steps.snowpark_execute_should_return_expected_value( + object_type="function", + identifier="hello_function('foo')", + expected_value='"Hello foo!"', + ) + + # Check if adding import triggers replace + _test_steps.package_should_build_proper_artifact( + "dummy_pkg_for_tests", "dummy_pkg_for_tests/shrubbery.py" + ) + _test_steps.package_should_upload_artifact_to_stage( + "dummy_pkg_for_tests.zip", STAGE_NAME + ) + + alter_snowflake_yml( + tmp_dir / "snowflake.yml", + parameter_path="snowpark.functions.0.imports", + value=["@dev_deployment/dummy_pkg_for_tests.zip"], + ) + + _test_steps.snowpark_deploy_should_finish_successfully_and_return( + additional_arguments=["--replace"], + expected_result=[ + { + "object": f"{database}.PUBLIC.hello_procedure(name string)", + "status": "packages updated", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.test()", + "status": "packages updated", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.hello_function(name string)", + "status": "definition updated", + "type": "function", + }, + ], + ) + + # Same file should be present, with addition of uploaded package + expected_files.append(f"{STAGE_NAME}/dummy_pkg_for_tests.zip") + + _test_steps.assert_that_only_these_files_are_staged_in_test_db( + *expected_files, stage_name=STAGE_NAME + ) + + # Grants are preserved after updates + + _test_steps.assert_that_object_has_expected_grant( + object_type="procedure", + object_name="hello_procedure(VARCHAR)", + expected_privillege="USAGE", + expected_role="test_role", + ) + + _test_steps.assert_that_object_has_expected_grant( + object_type="function", + object_name="hello_function(VARCHAR)", + expected_privillege="USAGE", + expected_role="test_role", + ) + + # Check if objects can be dropped + _test_steps.object_drop_should_finish_successfully( + object_type="procedure", identifier="hello_procedure(varchar)" + ) + _test_steps.object_drop_should_finish_successfully( + object_type="function", identifier="hello_function(varchar)" + ) + + _test_steps.object_show_should_return_no_data( + object_type="function", object_prefix="hello" + ) + _test_steps.object_show_should_return_no_data( + object_type="procedure", object_prefix="hello" + ) + + _test_steps.assert_that_only_these_files_are_staged_in_test_db( + *expected_files, stage_name=STAGE_NAME + ) + + +@pytest.mark.integration +def test_snowpark_flow_old_build( _test_steps, project_directory, alter_snowflake_yml, test_database ): database = test_database.upper() @@ -930,6 +1237,84 @@ def test_snowpark_aliases(project_directory, runner, _test_steps, test_database) @pytest.mark.integration def test_snowpark_flow_v2( + _test_steps, + project_directory, + alter_snowflake_yml, + test_database, + enable_snowpark_glob_support_feature_flag, +): + database = test_database.upper() + with project_directory("snowpark_v2") as tmp_dir: + _test_steps.snowpark_build_should_zip_files( + additional_files=[ + Path("output"), + Path("output") / "app_1.zip", + Path("output") / "app_2.zip", + Path("output") / "c.py", + ] + ) + _test_steps.snowpark_deploy_should_finish_successfully_and_return( + [ + { + "object": f"{database}.PUBLIC.hello_procedure(name string)", + "status": "created", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.test()", + "status": "created", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.hello_function(name string)", + "status": "created", + "type": "function", + }, + ] + ) + + _test_steps.assert_those_procedures_are_in_snowflake( + "hello_procedure(VARCHAR) RETURN VARCHAR" + ) + _test_steps.assert_those_functions_are_in_snowflake( + "hello_function(VARCHAR) RETURN VARCHAR" + ) + + _test_steps.assert_that_only_these_files_are_staged_in_test_db( + "stage_a/app_1.zip", + "stage_a/dependencies.zip", + stage_name="stage_a", + ) + + _test_steps.assert_that_only_these_files_are_staged_in_test_db( + f"{STAGE_NAME}/app_2.zip", + f"{STAGE_NAME}/c.py", + f"{STAGE_NAME}/dependencies.zip", + stage_name=STAGE_NAME, + ) + + # Created objects can be executed + _test_steps.snowpark_execute_should_return_expected_value( + object_type="procedure", + identifier="hello_procedure('foo')", + expected_value="Hello foo", + ) + + _test_steps.snowpark_execute_should_return_expected_value( + object_type="procedure", + identifier="test()", + expected_value="Test procedure", + ) + + _test_steps.snowpark_execute_should_return_expected_value( + object_type="function", + identifier="hello_function('foo')", + expected_value="Hello foo!", + ) + + +@pytest.mark.integration +def test_snowpark_flow_v2_old_build( _test_steps, project_directory, alter_snowflake_yml, test_database ): database = test_database.upper() @@ -997,6 +1382,50 @@ def test_snowpark_flow_v2( ) +@pytest.mark.integration +def test_snowpark_with_glob_patterns( + _test_steps, + project_directory, + alter_snowflake_yml, + test_database, + enable_snowpark_glob_support_feature_flag, +): + database = test_database.upper() + with project_directory("snowpark_glob_patterns"): + _test_steps.snowpark_build_should_zip_files( + additional_files=[ + Path("output"), + Path("output") / "app_1.zip", + Path("output") / "app_2.zip", + Path("output") / "e.py", + ] + ) + _test_steps.snowpark_deploy_should_finish_successfully_and_return( + [ + { + "object": f"{database}.PUBLIC.hello_procedure(name string)", + "status": "created", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.test()", + "status": "created", + "type": "procedure", + }, + { + "object": f"{database}.PUBLIC.hello_function(name string)", + "status": "created", + "type": "function", + }, + ] + ) + _test_steps.snowpark_execute_should_return_expected_value( + object_type="procedure", + identifier="hello_procedure('foo')", + expected_value="Hello foo" + "Test procedure", + ) + + @pytest.fixture def _test_setup( runner, diff --git a/tests_integration/testing_utils/snowpark_utils.py b/tests_integration/testing_utils/snowpark_utils.py index 9551dd5a78..33412f2d0f 100644 --- a/tests_integration/testing_utils/snowpark_utils.py +++ b/tests_integration/testing_utils/snowpark_utils.py @@ -24,6 +24,7 @@ from syrupy import SnapshotAssertion +from snowflake.cli.api.feature_flags import FeatureFlag from tests_integration.conftest import SnowCLIRunner from tests_integration.testing_utils import assert_that_result_is_error from tests_integration.testing_utils.assertions.test_file_assertions import ( @@ -170,7 +171,10 @@ def snowpark_build_should_zip_files( additional_files = [] if not no_dependencies: - additional_files.append(Path("dependencies.zip")) + if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): + additional_files.append(Path("output") / "dependencies.zip") + else: + additional_files.append(Path("dependencies.zip")) current_files = set(Path(".").glob("**/*")) result = self._setup.runner.invoke_with_connection_json( From 986d19803be221bc0483e0d3c29f07b47204d7e5 Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Thu, 21 Nov 2024 04:39:30 -0500 Subject: [PATCH 02/10] Fix broken v1 to v2 test on the feature branch (#1876) --- tests/helpers/__snapshots__/test_v1_to_v2.ambr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/helpers/__snapshots__/test_v1_to_v2.ambr b/tests/helpers/__snapshots__/test_v1_to_v2.ambr index d1e6e41d71..ddfd878c1b 100644 --- a/tests/helpers/__snapshots__/test_v1_to_v2.ambr +++ b/tests/helpers/__snapshots__/test_v1_to_v2.ambr @@ -188,7 +188,7 @@ type: string stage: dev_deployment artifacts: - - src: app + - src: app/ dest: my_snowpark_project type: procedure execute_as_caller: false @@ -212,7 +212,7 @@ runtime: '3.1' stage: dev_deployment artifacts: - - src: app + - src: app/ dest: my_snowpark_project type: function test_streamlit: @@ -225,9 +225,9 @@ pages_dir: None stage: streamlit artifacts: - - streamlit_app.py - - environment.yml - - pages + - src: streamlit_app.py + - src: environment.yml + - src: pages pkg: meta: role: pkg_role From 01e062a81ee82225cd5563ab9a567af74a702bf4 Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Mon, 25 Nov 2024 10:52:11 -0500 Subject: [PATCH 03/10] Moved native app artifacts logic to common API (#1877) --- .github/CODEOWNERS | 4 + .../cli/_plugins/nativeapp/artifacts.py | 628 +-------- .../setup/native_app_setup_processor.py | 2 +- .../codegen/snowpark/python_processor.py | 2 +- .../codegen/templates/templates_processor.py | 2 +- .../nativeapp/entities/application_package.py | 2 +- .../cli/_plugins/snowpark/commands.py | 3 +- src/snowflake/cli/_plugins/snowpark/zipper.py | 2 +- src/snowflake/cli/_plugins/stage/diff.py | 2 +- src/snowflake/cli/_plugins/stage/manager.py | 3 +- src/snowflake/cli/_plugins/stage/utils.py | 2 +- .../cli/_plugins/streamlit/manager.py | 3 +- .../cli/_plugins/workspace/commands.py | 2 +- src/snowflake/cli/api/artifacts/__init__.py | 13 + src/snowflake/cli/api/artifacts/bundle_map.py | 500 +++++++ src/snowflake/cli/api/artifacts/common.py | 78 ++ src/snowflake/cli/api/artifacts/utils.py | 51 + src/snowflake/cli/api/entities/utils.py | 6 +- src/snowflake/cli/api/utils/path_utils.py | 24 + tests/api/artifacts/test_bundle_map.py | 923 ++++++++++++ .../utils/__snapshots__/test_path_utils.ambr | 326 +++++ tests/api/utils/test_path_utils.py | 340 +++++ .../__snapshots__/test_artifacts.ambr | 325 ----- tests/nativeapp/fixtures.py | 2 +- tests/nativeapp/test_artifacts.py | 1241 +---------------- tests/nativeapp/test_manager.py | 2 +- tests/stage/test_diff.py | 2 +- 27 files changed, 2287 insertions(+), 2203 deletions(-) create mode 100644 src/snowflake/cli/api/artifacts/__init__.py create mode 100644 src/snowflake/cli/api/artifacts/bundle_map.py create mode 100644 src/snowflake/cli/api/artifacts/common.py create mode 100644 src/snowflake/cli/api/artifacts/utils.py create mode 100644 tests/api/artifacts/test_bundle_map.py create mode 100644 tests/api/utils/__snapshots__/test_path_utils.ambr create mode 100644 tests/api/utils/test_path_utils.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bc5af947d9..a38977373b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,10 +4,14 @@ **/__snapshots__/ @snowflakedb/snowcli @snowflakedb/nade **/test_data @snowflakedb/snowcli @snowflakedb/nade **/testing_utils/ @snowflakedb/snowcli @snowflakedb/nade +src/snowflake/cli/api/artifacts/ @snowflakedb/snowcli @snowflakedb/nade src/snowflake/cli/api/metrics.py @snowflakedb/snowcli @snowflakedb/nade src/snowflake/cli/api/project/definition_conversion.py @snowflakedb/snowcli @snowflakedb/nade src/snowflake/cli/api/utils/ @snowflakedb/snowcli @snowflakedb/nade +tests/api/artifacts/ @snowflakedb/snowcli @snowflakedb/nade +tests/api/metrics/ @snowflakedb/snowcli @snowflakedb/nade tests/api/test_metrics.py @snowflakedb/snowcli @snowflakedb/nade +tests/api/utils/ @snowflakedb/snowcli @snowflakedb/nade tests_common/__init__.py @snowflakedb/snowcli @snowflakedb/nade tests_common/conftest.py @snowflakedb/snowcli @snowflakedb/nade tests_common/deflake.py @snowflakedb/snowcli @snowflakedb/nade diff --git a/src/snowflake/cli/_plugins/nativeapp/artifacts.py b/src/snowflake/cli/_plugins/nativeapp/artifacts.py index 2ba28a446c..4e38faf9d6 100644 --- a/src/snowflake/cli/_plugins/nativeapp/artifacts.py +++ b/src/snowflake/cli/_plugins/nativeapp/artifacts.py @@ -14,642 +14,24 @@ from __future__ import annotations -import itertools import os from collections import namedtuple from pathlib import Path -from textwrap import dedent -from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional from click.exceptions import ClickException +from snowflake.cli.api.artifacts.bundle_map import BundleMap +from snowflake.cli.api.artifacts.common import ArtifactError, DeployRootError +from snowflake.cli.api.artifacts.utils import symlink_or_copy from snowflake.cli.api.cli_global_context import span from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.project.util import to_identifier from snowflake.cli.api.secure_path import SecurePath +from snowflake.cli.api.utils.path_utils import delete from yaml import safe_load -class DeployRootError(ClickException): - """ - The deploy root was incorrectly specified. - """ - - def __init__(self, msg: str): - super().__init__(msg) - - -class ArtifactError(ClickException): - """ - Could not parse source or destination artifact. - """ - - def __init__(self, msg: str): - super().__init__(msg) - - -class SourceNotFoundError(ClickException): - """ - No match was found for the specified source in the project directory - """ - - def __init__(self, src: Union[str, Path]): - super().__init__(f"{dedent(str(self.__doc__))}: {src}".strip()) - - -class TooManyFilesError(ClickException): - """ - Multiple file or directories were mapped to one output destination. - """ - - dest_path: Path - - def __init__(self, dest_path: Path): - super().__init__( - f"{dedent(str(self.__doc__))}\ndestination = {dest_path}".strip() - ) - self.dest_path = dest_path - - -class NotInDeployRootError(ClickException): - """ - The specified destination path is outside of the deploy root, or - would entirely replace it. This can happen when a relative path - with ".." is provided, or when "." is used as the destination - (use "./" instead to copy into the deploy root). - """ - - dest_path: Union[str, Path] - deploy_root: Path - src_path: Optional[Union[str, Path]] - - def __init__( - self, - *, - dest_path: Union[Path, str], - deploy_root: Path, - src_path: Optional[Union[str, Path]] = None, - ): - message = dedent(str(self.__doc__)) - message += f"\ndestination = {dest_path}" - message += f"\ndeploy root = {deploy_root}" - if src_path is not None: - message += f"""\nsource = {src_path}""" - super().__init__(message.strip()) - self.dest_path = dest_path - self.deploy_root = deploy_root - self.src_path = src_path - - -ArtifactPredicate = Callable[[Path, Path], bool] - - -class _ArtifactPathMap: - """ - A specialized version of an ordered multimap used to keep track of artifact - source-destination mappings. The mapping is bidirectional, so it can be queried - by source or destination paths. All paths manipulated by this class must be in - relative, canonical form (relative to the project or deploy roots, as appropriate). - """ - - def __init__(self, project_root: Path): - self._project_root = project_root - - # All (src,dest) pairs in inserting order, for iterating - self.__src_dest_pairs: List[Tuple[Path, Path]] = [] - # built-in dict instances are ordered as of Python 3.7 - self.__src_to_dest: Dict[Path, List[Path]] = {} - self.__dest_to_src: Dict[Path, Optional[Path]] = {} - - # This dictionary accumulates keys for each directory or file to be created in - # the deploy root for any artifact mapping rule being processed. This includes - # children of directories that are copied to the deploy root. Having this - # information available is critical to detect possible clashes between rules. - self._dest_is_dir: Dict[Path, bool] = {} - - def put(self, src: Path, dest: Path, dest_is_dir: bool) -> None: - """ - Adds a new source-destination mapping pair to this map, if necessary. Note that - this is internal logic that assumes that src-dest pairs have already been preprocessed - by the enclosing BundleMap (for example, only file -> file and - directory -> directory mappings are possible here due to the preprocessing step). - - Arguments: - src {Path} -- the source path, in canonical form. - dest {Path} -- the destination path, in canonical form. - dest_is_dir {bool} -- whether the destination path is a directory. - """ - # Both paths should be in canonical form - assert not src.is_absolute() - assert not dest.is_absolute() - - absolute_src = self._project_root / src - - current_source = self.__dest_to_src.get(dest) - src_is_dir = absolute_src.is_dir() - if dest_is_dir: - assert src_is_dir # file -> directory is not possible here given how rules are processed - - # directory -> directory - # Check that dest is currently unmapped - current_is_dir = self._dest_is_dir.get(dest, False) - if current_is_dir: - # mapping to an existing directory is not allowed - raise TooManyFilesError(dest) - else: - # file -> file - # Check that there is no previous mapping for the same file. - if current_source is not None and current_source != src: - # There is already a different source mapping to this destination - raise TooManyFilesError(dest) - - if src_is_dir: - # mark all subdirectories of this source as directories so that we can - # detect accidental clobbering - for root, _, files in os.walk(absolute_src, followlinks=True): - canonical_subdir = Path(root).relative_to(absolute_src) - canonical_dest_subdir = dest / canonical_subdir - self._update_dest_is_dir(canonical_dest_subdir, is_dir=True) - for f in files: - self._update_dest_is_dir(canonical_dest_subdir / f, is_dir=False) - - # make sure we check for dest_is_dir consistency regardless of whether the - # insertion happened. This update can fail, so we need to do it first to - # avoid applying partial updates to the underlying data storage. - self._update_dest_is_dir(dest, dest_is_dir) - - dests = self.__src_to_dest.setdefault(src, []) - if dest not in dests: - dests.append(dest) - self.__dest_to_src[dest] = src - self.__src_dest_pairs.append((src, dest)) - - def get_source(self, dest: Path) -> Optional[Path]: - """ - Returns the source path associated with the provided destination path, if any. - """ - return self.__dest_to_src.get(dest) - - def get_destinations(self, src: Path) -> Iterable[Path]: - """ - Returns all destination paths associated with the provided source path, in insertion order. - """ - return self.__src_to_dest.get(src, []) - - def all_sources(self) -> Iterable[Path]: - """ - Returns all source paths associated with this map, in insertion order. - """ - return self.__src_to_dest.keys() - - def is_empty(self) -> bool: - """ - Returns True if this map has no source-destination mappings. - """ - return len(self.__src_dest_pairs) == 0 - - def __iter__(self) -> Iterator[Tuple[Path, Path]]: - """ - Returns all (source, destination) pairs known to this map, in insertion order. - """ - return iter(self.__src_dest_pairs) - - def _update_dest_is_dir(self, dest: Path, is_dir: bool) -> None: - """ - Recursively marks seen destination paths as either files or folders, raising an error if any inconsistencies - from previous invocations of this method are encountered. - - Arguments: - dest {Path} -- the destination path, in canonical form. - is_dir {bool} -- whether the destination path is a directory. - """ - assert not dest.is_absolute() # dest must be in canonical relative form - - current_is_dir = self._dest_is_dir.get(dest, None) - if current_is_dir is not None and current_is_dir != is_dir: - raise ArtifactError(f"Conflicting type for destination path: {dest}") - - parent = dest.parent - if parent != dest: - self._update_dest_is_dir(parent, True) - - self._dest_is_dir[dest] = is_dir - - -class BundleMap: - """ - Computes the mapping between project directory artifacts (aka source artifacts) to their deploy root location - (aka destination artifact). This information is primarily used when bundling a native applications project. - - :param project_root: The root directory of the project and base for all relative paths. Must be an absolute path. - :param deploy_root: The directory where artifacts should be copied to. Must be an absolute path. - """ - - def __init__(self, *, project_root: Path, deploy_root: Path): - # If a relative path ends up here, it's a bug in the app and can lead to other - # subtle bugs as paths would be resolved relative to the current working directory. - assert ( - project_root.is_absolute() - ), f"Project root {project_root} must be an absolute path." - assert ( - deploy_root.is_absolute() - ), f"Deploy root {deploy_root} must be an absolute path." - - self._project_root: Path = resolve_without_follow(project_root) - self._deploy_root: Path = resolve_without_follow(deploy_root) - self._artifact_map = _ArtifactPathMap(project_root=self._project_root) - - def is_empty(self) -> bool: - return self._artifact_map.is_empty() - - def deploy_root(self) -> Path: - return self._deploy_root - - def project_root(self) -> Path: - return self._project_root - - def _add(self, src: Path, dest: Path, map_as_child: bool) -> None: - """ - Adds the specified artifact mapping rule to this map. - - Arguments: - src {Path} -- the source path - dest {Path} -- the destination path - map_as_child {bool} -- when True, the source will be added as a child of the specified destination. - """ - absolute_src = self._absolute_src(src) - absolute_dest = self._absolute_dest(dest, src_path=src) - dest_is_dir = absolute_src.is_dir() or map_as_child - - # Check for the special case of './' as a target ('.' is not allowed) - if absolute_dest == self._deploy_root and not map_as_child: - raise NotInDeployRootError( - dest_path=dest, deploy_root=self._deploy_root, src_path=src - ) - - if self._deploy_root in absolute_src.parents: - # ignore this item since it's in the deploy root. This can happen if the bundle map is created - # after the bundle step and a project is using rules that are not sufficiently constrained. - # Since the bundle step starts with deleting the deploy root, we wouldn't normally encounter this situation. - return - - canonical_src = self._canonical_src(src) - canonical_dest = self._canonical_dest(dest) - - if map_as_child: - # Make sure the destination is a child of the original, since this was requested - canonical_dest = canonical_dest / canonical_src.name - dest_is_dir = absolute_src.is_dir() - - self._artifact_map.put( - src=canonical_src, dest=canonical_dest, dest_is_dir=dest_is_dir - ) - - def _add_mapping(self, src: str, dest: Optional[str] = None): - """ - Adds the specified artifact rule to this instance. The source should be relative to the project directory. It - is interpreted as a file, directory or glob pattern. If the destination path is not specified, each source match - is mapped to an identical path in the deploy root. - """ - match_found = False - - src_path = Path(src) - if src_path.is_absolute(): - raise ArtifactError("Source path must be a relative path") - - for resolved_src in self._project_root.glob(src): - match_found = True - - if dest: - dest_stem = dest.rstrip("/") - if not dest_stem: - # handle '/' as the destination as a special case. This is because specifying only '/' as a - # a destination looks like '.' once all forwards slashes are stripped. If we don't handle it - # specially here, `dest: /` would incorrectly be allowed. - raise NotInDeployRootError( - dest_path=dest, - deploy_root=self._deploy_root, - src_path=resolved_src, - ) - dest_path = Path(dest.rstrip("/")) - if dest_path.is_absolute(): - raise ArtifactError("Destination path must be a relative path") - self._add(resolved_src, dest_path, specifies_directory(dest)) - else: - self._add( - resolved_src, - resolved_src.relative_to(self._project_root), - False, - ) - - if not match_found: - raise SourceNotFoundError(src) - - def add(self, mapping: PathMapping) -> None: - """ - Adds an artifact mapping rule to this instance. - """ - self._add_mapping(mapping.src, mapping.dest) - - def _expand_artifact_mapping( - self, - src: Path, - dest: Path, - absolute: bool = False, - expand_directories: bool = False, - predicate: ArtifactPredicate = lambda src, dest: True, - ) -> Iterator[Tuple[Path, Path]]: - """ - Expands the specified source-destination mapping according to the provided options. - The original mapping is yielded, followed by any expanded mappings derived from - it. - - Arguments: - src {Path} -- the source path - dest {Path} -- the destination path - absolute {bool} -- when True, all mappings will be yielded as absolute paths - expand_directories {bool} -- when True, child mappings are yielded if the source path is a directory. - predicate {ArtifactPredicate} -- when specified, only mappings satisfying this predicate will be yielded. - """ - canonical_src = self._canonical_src(src) - canonical_dest = self._canonical_dest(dest) - - absolute_src = self._absolute_src(canonical_src) - absolute_dest = self._absolute_dest(canonical_dest) - src_for_output = self._to_output_src(absolute_src, absolute) - dest_for_output = self._to_output_dest(absolute_dest, absolute) - - if predicate(src_for_output, dest_for_output): - yield src_for_output, dest_for_output - - if absolute_src.is_dir() and expand_directories: - # both src and dest are directories, and expanding directories was requested. Traverse src, and map each - # file to the dest directory - for root, subdirs, files in os.walk(absolute_src, followlinks=True): - relative_root = Path(root).relative_to(absolute_src) - for name in itertools.chain(subdirs, files): - src_file_for_output = src_for_output / relative_root / name - dest_file_for_output = dest_for_output / relative_root / name - if predicate(src_file_for_output, dest_file_for_output): - yield src_file_for_output, dest_file_for_output - - def all_mappings( - self, - absolute: bool = False, - expand_directories: bool = False, - predicate: ArtifactPredicate = lambda src, dest: True, - ) -> Iterator[Tuple[Path, Path]]: - """ - Yields a (src, dest) pair for each deployed artifact in the project. Each pair corresponds to a single file - in the project. Source directories are resolved as needed to resolve their contents. - - Arguments: - self: this instance - absolute (bool): Specifies whether the yielded paths should be joined with the project or deploy roots, - as appropriate. - expand_directories (bool): Specifies whether directory to directory mappings should be expanded to - resolve their contained files. - predicate (PathPredicate): If provided, the predicate is invoked with both the source path and the - destination path as arguments. Only pairs selected by the predicate are returned. - - Returns: - An iterator over all matching deployed artifacts. - """ - for src, dest in self._artifact_map: - for deployed_src, deployed_dest in self._expand_artifact_mapping( - src, - dest, - absolute=absolute, - expand_directories=expand_directories, - predicate=predicate, - ): - yield deployed_src, deployed_dest - - def to_deploy_paths(self, src: Path) -> List[Path]: - """ - Converts a source path to its corresponding deploy root path. If the input path is relative to the project root, - paths relative to the deploy root are returned. If the input path is absolute, absolute paths are returned. - - Note that the provided source path must be part of a mapping. If the source path is not part of any mapping, - an empty list is returned. For example, if `app/*` is specified as the source of a mapping, - `to_deploy_paths(Path("app"))` will not yield any result. - - Arguments: - src {Path} -- the source path within the project root, in canonical or absolute form. - - Returns: - The deploy root paths for the given source path, or an empty list if no such path exists. - """ - is_absolute = src.is_absolute() - - try: - absolute_src = self._absolute_src(src) - if not absolute_src.exists(): - return [] - canonical_src = self._canonical_src(absolute_src) - except ArtifactError: - # No mapping is possible for this src path - return [] - - output_destinations: List[Path] = [] - - # 1. Check for exact rule matches for this path - canonical_dests = self._artifact_map.get_destinations(canonical_src) - if canonical_dests: - for d in canonical_dests: - output_destinations.append(self._to_output_dest(d, is_absolute)) - - # 2. Check for any matches to parent directories for this path that would - # cause this path to be part of the recursive copy - canonical_parent = canonical_src.parent - canonical_parent_dests = self.to_deploy_paths(canonical_parent) - if canonical_parent_dests: - canonical_child = canonical_src.relative_to(canonical_parent) - for d in canonical_parent_dests: - output_destinations.append( - self._to_output_dest(d / canonical_child, is_absolute) - ) - - return output_destinations - - def all_sources(self, absolute: bool = False) -> Iterator[Path]: - """ - Yields each registered artifact source in the project. - - Arguments: - self: this instance - absolute (bool): Specifies whether the yielded paths should be joined with the absolute project root. - Returns: - An iterator over all artifact mapping source paths. - """ - for src in self._artifact_map.all_sources(): - yield self._to_output_src(src, absolute) - - def to_project_path(self, dest: Path) -> Optional[Path]: - """ - Converts a deploy root path to its corresponding project source path. If the input path is relative to the - deploy root, a path relative to the project root is returned. If the input path is absolute, an absolute path is - returned. - - Arguments: - dest {Path} -- the destination path within the deploy root, in canonical or absolute form. - - Returns: - The project root path for the given deploy root path, or None if no such path exists. - """ - is_absolute = dest.is_absolute() - try: - canonical_dest = self._canonical_dest(dest) - except NotInDeployRootError: - # No mapping possible for the dest path - return None - - # 1. Look for an exact rule matching this path. If we find any, then - # stop searching. This is because each destination path can only originate - # from a single source (however, one source can be copied to multiple destinations). - canonical_src = self._artifact_map.get_source(canonical_dest) - if canonical_src is not None: - return self._to_output_src(canonical_src, is_absolute) - - # 2. No exact match was found, look for a match for parent directories of this - # path, recursively. Stop when a match is found - canonical_parent = canonical_dest.parent - if canonical_parent == canonical_dest: - return None - canonical_parent_src = self.to_project_path(canonical_parent) - if canonical_parent_src is not None: - canonical_child = canonical_dest.relative_to(canonical_parent) - canonical_child_candidate = canonical_parent_src / canonical_child - if self._absolute_src(canonical_child_candidate).exists(): - return self._to_output_src(canonical_child_candidate, is_absolute) - - # No mapping for this destination path - return None - - def _absolute_src(self, src: Path) -> Path: - if src.is_absolute(): - resolved_src = resolve_without_follow(src) - else: - resolved_src = resolve_without_follow(self._project_root / src) - if self._project_root not in resolved_src.parents: - raise ArtifactError( - f"Source is not in the project root: {src}, root={self._project_root}" - ) - return resolved_src - - def _absolute_dest(self, dest: Path, src_path: Optional[Path] = None) -> Path: - if dest.is_absolute(): - resolved_dest = resolve_without_follow(dest) - else: - resolved_dest = resolve_without_follow(self._deploy_root / dest) - if ( - self._deploy_root != resolved_dest - and self._deploy_root not in resolved_dest.parents - ): - raise NotInDeployRootError( - dest_path=dest, deploy_root=self._deploy_root, src_path=src_path - ) - - return resolved_dest - - def _canonical_src(self, src: Path) -> Path: - """ - Returns the canonical version of a source path, relative to the project root. - """ - absolute_src = self._absolute_src(src) - return absolute_src.relative_to(self._project_root) - - def _canonical_dest(self, dest: Path) -> Path: - """ - Returns the canonical version of a destination path, relative to the deploy root. - """ - absolute_dest = self._absolute_dest(dest) - return absolute_dest.relative_to(self._deploy_root) - - def _to_output_dest(self, dest: Path, absolute: bool) -> Path: - return self._absolute_dest(dest) if absolute else self._canonical_dest(dest) - - def _to_output_src(self, src: Path, absolute: bool) -> Path: - return self._absolute_src(src) if absolute else self._canonical_src(src) - - -def specifies_directory(s: str) -> bool: - """ - Does the path (as seen from the project definition) refer to - a directory? For destination paths, we enforce the usage of a - trailing forward slash (/). Note that we use the forward slash - even on Windows so that snowflake.yml can be shared between OSes. - - This means that to put a file in the root of the stage, we need - to specify "./" as its destination, or omit it (but only if the - file already lives in the project root). - """ - return s.endswith("/") - - -def delete(path: Path) -> None: - """ - Obliterates whatever is at the given path, or is a no-op if the - given path does not represent a file or directory that exists. - """ - spath = SecurePath(path) - if spath.path.is_file(): - spath.unlink() # remove the file - elif spath.path.is_dir(): - spath.rmdir(recursive=True) # remove dir and all contains - - -def symlink_or_copy(src: Path, dst: Path, deploy_root: Path) -> None: - """ - Symlinks files from src to dst. If the src contains parent directories, then copies the empty directory shell to the deploy root. - The directory hierarchy above dst is created if any of those directories do not exist. - """ - ssrc = SecurePath(src) - sdst = SecurePath(dst) - sdst.parent.mkdir(parents=True, exist_ok=True) - - # Verify that the mapping isn't accidentally trying to create a file in the project source through symlinks. - # We need to ensure we're resolving symlinks for this check to be effective. - # We are unlikely to hit this if calling the function through bundle map, keeping it here for other future use cases outside bundle. - resolved_dst = dst.resolve() - resolved_deploy_root = deploy_root.resolve() - dst_is_deploy_root = resolved_deploy_root == resolved_dst - if (not dst_is_deploy_root) and (resolved_deploy_root not in resolved_dst.parents): - raise NotInDeployRootError(dest_path=dst, deploy_root=deploy_root, src_path=src) - - absolute_src = resolve_without_follow(src) - if absolute_src.is_file(): - delete(dst) - try: - os.symlink(absolute_src, dst) - except OSError: - ssrc.copy(dst) - else: - # 1. Create a new directory in the deploy root - dst.mkdir(exist_ok=True) - # 2. For all children of src, create their counterparts in dst now that it exists - for root, _, files in sorted(os.walk(absolute_src, followlinks=True)): - relative_root = Path(root).relative_to(absolute_src) - absolute_root_in_deploy = Path(dst, relative_root) - absolute_root_in_deploy.mkdir(parents=True, exist_ok=True) - for file in sorted(files): - absolute_file_in_project = Path(absolute_src, relative_root, file) - absolute_file_in_deploy = Path(absolute_root_in_deploy, file) - symlink_or_copy( - src=absolute_file_in_project, - dst=absolute_file_in_deploy, - deploy_root=deploy_root, - ) - - -def resolve_without_follow(path: Path) -> Path: - """ - Resolves a Path to an absolute version of itself, without following - symlinks like Path.resolve() does. - """ - return Path(os.path.abspath(path)) - - @span("bundle") def build_bundle( project_root: Path, diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py index b643f66304..c62743eab0 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py @@ -23,7 +23,6 @@ import yaml from click import ClickException from snowflake.cli._plugins.nativeapp.artifacts import ( - BundleMap, find_manifest_file, find_setup_script_file, ) @@ -38,6 +37,7 @@ ) from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag from snowflake.cli._plugins.stage.diff import to_stage_path +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( PathMapping, diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py index 58a9eb2baa..2fc58cd4ab 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py @@ -22,7 +22,6 @@ from pydantic import ValidationError from snowflake.cli._plugins.nativeapp.artifacts import ( - BundleMap, find_setup_script_file, ) from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import ( @@ -48,6 +47,7 @@ NativeAppExtensionFunction, ) from snowflake.cli._plugins.stage.diff import to_stage_path +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.metrics import CLICounterField diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index b6984f67c2..ff1ee55084 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -18,11 +18,11 @@ from typing import Any, Optional import jinja2 -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import ( ArtifactProcessor, ) from snowflake.cli._plugins.nativeapp.exceptions import InvalidTemplateInFileError +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.metrics import CLICounterField diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index fc5a675fb6..bc6e2805a2 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -12,7 +12,6 @@ from pydantic import Field, field_validator from snowflake.cli._plugins.connection.util import UIParameter from snowflake.cli._plugins.nativeapp.artifacts import ( - BundleMap, VersionInfo, build_bundle, find_setup_script_file, @@ -64,6 +63,7 @@ StreamlitEntityModel, ) from snowflake.cli._plugins.workspace.context import ActionContext +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.cli_global_context import span from snowflake.cli.api.entities.common import ( EntityBase, diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py index cbe9cea568..25e73eda2a 100644 --- a/src/snowflake/cli/_plugins/snowpark/commands.py +++ b/src/snowflake/cli/_plugins/snowpark/commands.py @@ -20,7 +20,6 @@ import typer from click import ClickException, UsageError -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap, symlink_or_copy from snowflake.cli._plugins.object.commands import ( describe as object_describe, ) @@ -62,6 +61,8 @@ ) from snowflake.cli._plugins.snowpark.zipper import zip_dir, zip_dir_using_bundle_map from snowflake.cli._plugins.stage.manager import StageManager +from snowflake.cli.api.artifacts.bundle_map import BundleMap +from snowflake.cli.api.artifacts.utils import symlink_or_copy from snowflake.cli.api.cli_global_context import ( get_cli_context, ) diff --git a/src/snowflake/cli/_plugins/snowpark/zipper.py b/src/snowflake/cli/_plugins/snowpark/zipper.py index 34889680d0..55fe74ca2a 100644 --- a/src/snowflake/cli/_plugins/snowpark/zipper.py +++ b/src/snowflake/cli/_plugins/snowpark/zipper.py @@ -20,7 +20,7 @@ from typing import Dict, List, Literal from zipfile import ZIP_DEFLATED, ZipFile -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.console import cli_console log = logging.getLogger(__name__) diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index ab3b2e0d91..1ada3acbbf 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -19,7 +19,7 @@ from pathlib import Path, PurePosixPath from typing import Collection, Dict, List, Optional, Tuple -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.exceptions import ( SnowflakeSQLExecutionError, ) diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index 71b4e18cdd..6e441fe80c 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -45,7 +45,7 @@ from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.cli.api.stage_path import StagePath -from snowflake.cli.api.utils.path_utils import path_resolver +from snowflake.cli.api.utils.path_utils import path_resolver, resolve_without_follow from snowflake.connector import DictCursor, ProgrammingError from snowflake.connector.cursor import SnowflakeCursor @@ -345,7 +345,6 @@ def put( @staticmethod def _symlink_or_copy(source_root: Path, source_file_or_dir: Path, dest_dir: Path): - from snowflake.cli._plugins.nativeapp.artifacts import resolve_without_follow absolute_src = resolve_without_follow(source_file_or_dir) dest_path = dest_dir / source_file_or_dir.relative_to(source_root) diff --git a/src/snowflake/cli/_plugins/stage/utils.py b/src/snowflake/cli/_plugins/stage/utils.py index a3c440f31b..bce09c8434 100644 --- a/src/snowflake/cli/_plugins/stage/utils.py +++ b/src/snowflake/cli/_plugins/stage/utils.py @@ -1,11 +1,11 @@ from typing import Optional -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.stage.diff import ( DiffResult, _to_diff_line, _to_src_dest_pair, ) +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.console import cli_console as cc diff --git a/src/snowflake/cli/_plugins/streamlit/manager.py b/src/snowflake/cli/_plugins/streamlit/manager.py index 1ec889089a..a83f553df2 100644 --- a/src/snowflake/cli/_plugins/streamlit/manager.py +++ b/src/snowflake/cli/_plugins/streamlit/manager.py @@ -24,7 +24,6 @@ MissingConnectionRegionError, make_snowsight_url, ) -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap, symlink_or_copy from snowflake.cli._plugins.object.manager import ObjectManager from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli._plugins.streamlit.streamlit_entity_model import ( @@ -33,6 +32,8 @@ from snowflake.cli._plugins.streamlit.streamlit_project_paths import ( StreamlitProjectPaths, ) +from snowflake.cli.api.artifacts.bundle_map import BundleMap +from snowflake.cli.api.artifacts.utils import symlink_or_copy from snowflake.cli.api.commands.experimental_behaviour import ( experimental_behaviour_enabled, ) diff --git a/src/snowflake/cli/_plugins/workspace/commands.py b/src/snowflake/cli/_plugins/workspace/commands.py index e09078df45..928d0a4876 100644 --- a/src/snowflake/cli/_plugins/workspace/commands.py +++ b/src/snowflake/cli/_plugins/workspace/commands.py @@ -22,13 +22,13 @@ import typer import yaml -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.nativeapp.common_flags import ( ForceOption, InteractiveOption, ValidateOption, ) from snowflake.cli._plugins.workspace.manager import WorkspaceManager +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.commands.decorators import with_project_definition from snowflake.cli.api.commands.snow_typer import SnowTyperFactory diff --git a/src/snowflake/cli/api/artifacts/__init__.py b/src/snowflake/cli/api/artifacts/__init__.py new file mode 100644 index 0000000000..ada0a4e13d --- /dev/null +++ b/src/snowflake/cli/api/artifacts/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/snowflake/cli/api/artifacts/bundle_map.py b/src/snowflake/cli/api/artifacts/bundle_map.py new file mode 100644 index 0000000000..93089f71e2 --- /dev/null +++ b/src/snowflake/cli/api/artifacts/bundle_map.py @@ -0,0 +1,500 @@ +from __future__ import annotations + +import itertools +import os +from pathlib import Path +from typing import Callable, Dict, Iterable, Iterator, List, Optional, Tuple + +from snowflake.cli.api.artifacts.common import ( + ArtifactError, + NotInDeployRootError, + SourceNotFoundError, + TooManyFilesError, +) +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.utils.path_utils import resolve_without_follow + +ArtifactPredicate = Callable[[Path, Path], bool] + + +def _specifies_directory(s: str) -> bool: + """ + Does the path (as seen from the project definition) refer to + a directory? For destination paths, we enforce the usage of a + trailing forward slash (/). Note that we use the forward slash + even on Windows so that snowflake.yml can be shared between OSes. + + This means that to put a file in the root of the stage, we need + to specify "./" as its destination, or omit it (but only if the + file already lives in the project root). + """ + return s.endswith("/") + + +class BundleMap: + """ + Computes the mapping between project directory artifacts (aka source artifacts) to their deploy root location + (aka destination artifact). This information is primarily used when bundling a native applications project. + + :param project_root: The root directory of the project and base for all relative paths. Must be an absolute path. + :param deploy_root: The directory where artifacts should be copied to. Must be an absolute path. + """ + + def __init__(self, *, project_root: Path, deploy_root: Path): + # If a relative path ends up here, it's a bug in the app and can lead to other + # subtle bugs as paths would be resolved relative to the current working directory. + assert ( + project_root.is_absolute() + ), f"Project root {project_root} must be an absolute path." + assert ( + deploy_root.is_absolute() + ), f"Deploy root {deploy_root} must be an absolute path." + + self._project_root: Path = resolve_without_follow(project_root) + self._deploy_root: Path = resolve_without_follow(deploy_root) + self._artifact_map = _ArtifactPathMap(project_root=self._project_root) + + def is_empty(self) -> bool: + return self._artifact_map.is_empty() + + def deploy_root(self) -> Path: + return self._deploy_root + + def project_root(self) -> Path: + return self._project_root + + def _add(self, src: Path, dest: Path, map_as_child: bool) -> None: + """ + Adds the specified artifact mapping rule to this map. + + Arguments: + src {Path} -- the source path + dest {Path} -- the destination path + map_as_child {bool} -- when True, the source will be added as a child of the specified destination. + """ + absolute_src = self._absolute_src(src) + absolute_dest = self._absolute_dest(dest, src_path=src) + dest_is_dir = absolute_src.is_dir() or map_as_child + + # Check for the special case of './' as a target ('.' is not allowed) + if absolute_dest == self._deploy_root and not map_as_child: + raise NotInDeployRootError( + dest_path=dest, deploy_root=self._deploy_root, src_path=src + ) + + if self._deploy_root in absolute_src.parents: + # ignore this item since it's in the deploy root. This can happen if the bundle map is created + # after the bundle step and a project is using rules that are not sufficiently constrained. + # Since the bundle step starts with deleting the deploy root, we wouldn't normally encounter this situation. + return + + canonical_src = self._canonical_src(src) + canonical_dest = self._canonical_dest(dest) + + if map_as_child: + # Make sure the destination is a child of the original, since this was requested + canonical_dest = canonical_dest / canonical_src.name + dest_is_dir = absolute_src.is_dir() + + self._artifact_map.put( + src=canonical_src, dest=canonical_dest, dest_is_dir=dest_is_dir + ) + + def _add_mapping(self, src: str, dest: Optional[str] = None): + """ + Adds the specified artifact rule to this instance. The source should be relative to the project directory. It + is interpreted as a file, directory or glob pattern. If the destination path is not specified, each source match + is mapped to an identical path in the deploy root. + """ + match_found = False + + src_path = Path(src) + if src_path.is_absolute(): + raise ArtifactError("Source path must be a relative path") + + for resolved_src in self._project_root.glob(src): + match_found = True + + if dest: + dest_stem = dest.rstrip("/") + if not dest_stem: + # handle '/' as the destination as a special case. This is because specifying only '/' as a + # a destination looks like '.' once all forwards slashes are stripped. If we don't handle it + # specially here, `dest: /` would incorrectly be allowed. + raise NotInDeployRootError( + dest_path=dest, + deploy_root=self._deploy_root, + src_path=resolved_src, + ) + dest_path = Path(dest.rstrip("/")) + if dest_path.is_absolute(): + raise ArtifactError("Destination path must be a relative path") + self._add(resolved_src, dest_path, _specifies_directory(dest)) + else: + self._add( + resolved_src, + resolved_src.relative_to(self._project_root), + False, + ) + + if not match_found: + raise SourceNotFoundError(src) + + def add(self, mapping: PathMapping) -> None: + """ + Adds an artifact mapping rule to this instance. + """ + self._add_mapping(mapping.src, mapping.dest) + + def _expand_artifact_mapping( + self, + src: Path, + dest: Path, + absolute: bool = False, + expand_directories: bool = False, + predicate: ArtifactPredicate = lambda src, dest: True, + ) -> Iterator[Tuple[Path, Path]]: + """ + Expands the specified source-destination mapping according to the provided options. + The original mapping is yielded, followed by any expanded mappings derived from + it. + + Arguments: + src {Path} -- the source path + dest {Path} -- the destination path + absolute {bool} -- when True, all mappings will be yielded as absolute paths + expand_directories {bool} -- when True, child mappings are yielded if the source path is a directory. + predicate {ArtifactPredicate} -- when specified, only mappings satisfying this predicate will be yielded. + """ + canonical_src = self._canonical_src(src) + canonical_dest = self._canonical_dest(dest) + + absolute_src = self._absolute_src(canonical_src) + absolute_dest = self._absolute_dest(canonical_dest) + src_for_output = self._to_output_src(absolute_src, absolute) + dest_for_output = self._to_output_dest(absolute_dest, absolute) + + if predicate(src_for_output, dest_for_output): + yield src_for_output, dest_for_output + + if absolute_src.is_dir() and expand_directories: + # both src and dest are directories, and expanding directories was requested. Traverse src, and map each + # file to the dest directory + for root, subdirs, files in os.walk(absolute_src, followlinks=True): + relative_root = Path(root).relative_to(absolute_src) + for name in itertools.chain(subdirs, files): + src_file_for_output = src_for_output / relative_root / name + dest_file_for_output = dest_for_output / relative_root / name + if predicate(src_file_for_output, dest_file_for_output): + yield src_file_for_output, dest_file_for_output + + def all_mappings( + self, + absolute: bool = False, + expand_directories: bool = False, + predicate: ArtifactPredicate = lambda src, dest: True, + ) -> Iterator[Tuple[Path, Path]]: + """ + Yields a (src, dest) pair for each deployed artifact in the project. Each pair corresponds to a single file + in the project. Source directories are resolved as needed to resolve their contents. + + Arguments: + self: this instance + absolute (bool): Specifies whether the yielded paths should be joined with the project or deploy roots, + as appropriate. + expand_directories (bool): Specifies whether directory to directory mappings should be expanded to + resolve their contained files. + predicate (PathPredicate): If provided, the predicate is invoked with both the source path and the + destination path as arguments. Only pairs selected by the predicate are returned. + + Returns: + An iterator over all matching deployed artifacts. + """ + for src, dest in self._artifact_map: + for deployed_src, deployed_dest in self._expand_artifact_mapping( + src, + dest, + absolute=absolute, + expand_directories=expand_directories, + predicate=predicate, + ): + yield deployed_src, deployed_dest + + def to_deploy_paths(self, src: Path) -> List[Path]: + """ + Converts a source path to its corresponding deploy root path. If the input path is relative to the project root, + paths relative to the deploy root are returned. If the input path is absolute, absolute paths are returned. + + Note that the provided source path must be part of a mapping. If the source path is not part of any mapping, + an empty list is returned. For example, if `app/*` is specified as the source of a mapping, + `to_deploy_paths(Path("app"))` will not yield any result. + + Arguments: + src {Path} -- the source path within the project root, in canonical or absolute form. + + Returns: + The deploy root paths for the given source path, or an empty list if no such path exists. + """ + is_absolute = src.is_absolute() + + try: + absolute_src = self._absolute_src(src) + if not absolute_src.exists(): + return [] + canonical_src = self._canonical_src(absolute_src) + except ArtifactError: + # No mapping is possible for this src path + return [] + + output_destinations: List[Path] = [] + + # 1. Check for exact rule matches for this path + canonical_dests = self._artifact_map.get_destinations(canonical_src) + if canonical_dests: + for d in canonical_dests: + output_destinations.append(self._to_output_dest(d, is_absolute)) + + # 2. Check for any matches to parent directories for this path that would + # cause this path to be part of the recursive copy + canonical_parent = canonical_src.parent + canonical_parent_dests = self.to_deploy_paths(canonical_parent) + if canonical_parent_dests: + canonical_child = canonical_src.relative_to(canonical_parent) + for d in canonical_parent_dests: + output_destinations.append( + self._to_output_dest(d / canonical_child, is_absolute) + ) + + return output_destinations + + def all_sources(self, absolute: bool = False) -> Iterator[Path]: + """ + Yields each registered artifact source in the project. + + Arguments: + self: this instance + absolute (bool): Specifies whether the yielded paths should be joined with the absolute project root. + Returns: + An iterator over all artifact mapping source paths. + """ + for src in self._artifact_map.all_sources(): + yield self._to_output_src(src, absolute) + + def to_project_path(self, dest: Path) -> Optional[Path]: + """ + Converts a deploy root path to its corresponding project source path. If the input path is relative to the + deploy root, a path relative to the project root is returned. If the input path is absolute, an absolute path is + returned. + + Arguments: + dest {Path} -- the destination path within the deploy root, in canonical or absolute form. + + Returns: + The project root path for the given deploy root path, or None if no such path exists. + """ + is_absolute = dest.is_absolute() + try: + canonical_dest = self._canonical_dest(dest) + except NotInDeployRootError: + # No mapping possible for the dest path + return None + + # 1. Look for an exact rule matching this path. If we find any, then + # stop searching. This is because each destination path can only originate + # from a single source (however, one source can be copied to multiple destinations). + canonical_src = self._artifact_map.get_source(canonical_dest) + if canonical_src is not None: + return self._to_output_src(canonical_src, is_absolute) + + # 2. No exact match was found, look for a match for parent directories of this + # path, recursively. Stop when a match is found + canonical_parent = canonical_dest.parent + if canonical_parent == canonical_dest: + return None + canonical_parent_src = self.to_project_path(canonical_parent) + if canonical_parent_src is not None: + canonical_child = canonical_dest.relative_to(canonical_parent) + canonical_child_candidate = canonical_parent_src / canonical_child + if self._absolute_src(canonical_child_candidate).exists(): + return self._to_output_src(canonical_child_candidate, is_absolute) + + # No mapping for this destination path + return None + + def _absolute_src(self, src: Path) -> Path: + if src.is_absolute(): + resolved_src = resolve_without_follow(src) + else: + resolved_src = resolve_without_follow(self._project_root / src) + if self._project_root not in resolved_src.parents: + raise ArtifactError( + f"Source is not in the project root: {src}, root={self._project_root}" + ) + return resolved_src + + def _absolute_dest(self, dest: Path, src_path: Optional[Path] = None) -> Path: + if dest.is_absolute(): + resolved_dest = resolve_without_follow(dest) + else: + resolved_dest = resolve_without_follow(self._deploy_root / dest) + if ( + self._deploy_root != resolved_dest + and self._deploy_root not in resolved_dest.parents + ): + raise NotInDeployRootError( + dest_path=dest, deploy_root=self._deploy_root, src_path=src_path + ) + + return resolved_dest + + def _canonical_src(self, src: Path) -> Path: + """ + Returns the canonical version of a source path, relative to the project root. + """ + absolute_src = self._absolute_src(src) + return absolute_src.relative_to(self._project_root) + + def _canonical_dest(self, dest: Path) -> Path: + """ + Returns the canonical version of a destination path, relative to the deploy root. + """ + absolute_dest = self._absolute_dest(dest) + return absolute_dest.relative_to(self._deploy_root) + + def _to_output_dest(self, dest: Path, absolute: bool) -> Path: + return self._absolute_dest(dest) if absolute else self._canonical_dest(dest) + + def _to_output_src(self, src: Path, absolute: bool) -> Path: + return self._absolute_src(src) if absolute else self._canonical_src(src) + + +class _ArtifactPathMap: + """ + A specialized version of an ordered multimap used to keep track of artifact + source-destination mappings. The mapping is bidirectional, so it can be queried + by source or destination paths. All paths manipulated by this class must be in + relative, canonical form (relative to the project or deploy roots, as appropriate). + """ + + def __init__(self, project_root: Path): + self._project_root = project_root + + # All (src,dest) pairs in inserting order, for iterating + self.__src_dest_pairs: List[Tuple[Path, Path]] = [] + # built-in dict instances are ordered as of Python 3.7 + self.__src_to_dest: Dict[Path, List[Path]] = {} + self.__dest_to_src: Dict[Path, Optional[Path]] = {} + + # This dictionary accumulates keys for each directory or file to be created in + # the deploy root for any artifact mapping rule being processed. This includes + # children of directories that are copied to the deploy root. Having this + # information available is critical to detect possible clashes between rules. + self._dest_is_dir: Dict[Path, bool] = {} + + def put(self, src: Path, dest: Path, dest_is_dir: bool) -> None: + """ + Adds a new source-destination mapping pair to this map, if necessary. Note that + this is internal logic that assumes that src-dest pairs have already been preprocessed + by the enclosing BundleMap (for example, only file -> file and + directory -> directory mappings are possible here due to the preprocessing step). + + Arguments: + src {Path} -- the source path, in canonical form. + dest {Path} -- the destination path, in canonical form. + dest_is_dir {bool} -- whether the destination path is a directory. + """ + # Both paths should be in canonical form + assert not src.is_absolute() + assert not dest.is_absolute() + + absolute_src = self._project_root / src + + current_source = self.__dest_to_src.get(dest) + src_is_dir = absolute_src.is_dir() + if dest_is_dir: + assert src_is_dir # file -> directory is not possible here given how rules are processed + + # directory -> directory + # Check that dest is currently unmapped + current_is_dir = self._dest_is_dir.get(dest, False) + if current_is_dir: + # mapping to an existing directory is not allowed + raise TooManyFilesError(dest) + else: + # file -> file + # Check that there is no previous mapping for the same file. + if current_source is not None and current_source != src: + # There is already a different source mapping to this destination + raise TooManyFilesError(dest) + + if src_is_dir: + # mark all subdirectories of this source as directories so that we can + # detect accidental clobbering + for root, _, files in os.walk(absolute_src, followlinks=True): + canonical_subdir = Path(root).relative_to(absolute_src) + canonical_dest_subdir = dest / canonical_subdir + self._update_dest_is_dir(canonical_dest_subdir, is_dir=True) + for f in files: + self._update_dest_is_dir(canonical_dest_subdir / f, is_dir=False) + + # make sure we check for dest_is_dir consistency regardless of whether the + # insertion happened. This update can fail, so we need to do it first to + # avoid applying partial updates to the underlying data storage. + self._update_dest_is_dir(dest, dest_is_dir) + + dests = self.__src_to_dest.setdefault(src, []) + if dest not in dests: + dests.append(dest) + self.__dest_to_src[dest] = src + self.__src_dest_pairs.append((src, dest)) + + def get_source(self, dest: Path) -> Optional[Path]: + """ + Returns the source path associated with the provided destination path, if any. + """ + return self.__dest_to_src.get(dest) + + def get_destinations(self, src: Path) -> Iterable[Path]: + """ + Returns all destination paths associated with the provided source path, in insertion order. + """ + return self.__src_to_dest.get(src, []) + + def all_sources(self) -> Iterable[Path]: + """ + Returns all source paths associated with this map, in insertion order. + """ + return self.__src_to_dest.keys() + + def is_empty(self) -> bool: + """ + Returns True if this map has no source-destination mappings. + """ + return len(self.__src_dest_pairs) == 0 + + def __iter__(self) -> Iterator[Tuple[Path, Path]]: + """ + Returns all (source, destination) pairs known to this map, in insertion order. + """ + return iter(self.__src_dest_pairs) + + def _update_dest_is_dir(self, dest: Path, is_dir: bool) -> None: + """ + Recursively marks seen destination paths as either files or folders, raising an error if any inconsistencies + from previous invocations of this method are encountered. + + Arguments: + dest {Path} -- the destination path, in canonical form. + is_dir {bool} -- whether the destination path is a directory. + """ + assert not dest.is_absolute() # dest must be in canonical relative form + + current_is_dir = self._dest_is_dir.get(dest, None) + if current_is_dir is not None and current_is_dir != is_dir: + raise ArtifactError(f"Conflicting type for destination path: {dest}") + + parent = dest.parent + if parent != dest: + self._update_dest_is_dir(parent, True) + + self._dest_is_dir[dest] = is_dir diff --git a/src/snowflake/cli/api/artifacts/common.py b/src/snowflake/cli/api/artifacts/common.py new file mode 100644 index 0000000000..c2c4043ddf --- /dev/null +++ b/src/snowflake/cli/api/artifacts/common.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from pathlib import Path +from textwrap import dedent +from typing import Optional, Union + +from click import ClickException + + +class DeployRootError(ClickException): + """ + The deploy root was incorrectly specified. + """ + + def __init__(self, msg: str): + super().__init__(msg) + + +class ArtifactError(ClickException): + """ + Could not parse source or destination artifact. + """ + + def __init__(self, msg: str): + super().__init__(msg) + + +class SourceNotFoundError(ClickException): + """ + No match was found for the specified source in the project directory + """ + + def __init__(self, src: Union[str, Path]): + super().__init__(f"{dedent(str(self.__doc__))}: {src}".strip()) + + +class TooManyFilesError(ClickException): + """ + Multiple file or directories were mapped to one output destination. + """ + + dest_path: Path + + def __init__(self, dest_path: Path): + super().__init__( + f"{dedent(str(self.__doc__))}\ndestination = {dest_path}".strip() + ) + self.dest_path = dest_path + + +class NotInDeployRootError(ClickException): + """ + The specified destination path is outside of the deploy root, or + would entirely replace it. This can happen when a relative path + with ".." is provided, or when "." is used as the destination + (use "./" instead to copy into the deploy root). + """ + + dest_path: Union[str, Path] + deploy_root: Path + src_path: Optional[Union[str, Path]] + + def __init__( + self, + *, + dest_path: Union[Path, str], + deploy_root: Path, + src_path: Optional[Union[str, Path]] = None, + ): + message = dedent(str(self.__doc__)) + message += f"\ndestination = {dest_path}" + message += f"\ndeploy root = {deploy_root}" + if src_path is not None: + message += f"""\nsource = {src_path}""" + super().__init__(message.strip()) + self.dest_path = dest_path + self.deploy_root = deploy_root + self.src_path = src_path diff --git a/src/snowflake/cli/api/artifacts/utils.py b/src/snowflake/cli/api/artifacts/utils.py new file mode 100644 index 0000000000..cbc2b9edb0 --- /dev/null +++ b/src/snowflake/cli/api/artifacts/utils.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from snowflake.cli.api.artifacts.common import NotInDeployRootError +from snowflake.cli.api.secure_path import SecurePath +from snowflake.cli.api.utils.path_utils import delete, resolve_without_follow + + +def symlink_or_copy(src: Path, dst: Path, deploy_root: Path) -> None: + """ + Symlinks files from src to dst. If the src contains parent directories, then copies the empty directory shell to the deploy root. + The directory hierarchy above dst is created if any of those directories do not exist. + """ + ssrc = SecurePath(src) + sdst = SecurePath(dst) + sdst.parent.mkdir(parents=True, exist_ok=True) + + # Verify that the mapping isn't accidentally trying to create a file in the project source through symlinks. + # We need to ensure we're resolving symlinks for this check to be effective. + # We are unlikely to hit this if calling the function through bundle map, keeping it here for other future use cases outside bundle. + resolved_dst = dst.resolve() + resolved_deploy_root = deploy_root.resolve() + dst_is_deploy_root = resolved_deploy_root == resolved_dst + if (not dst_is_deploy_root) and (resolved_deploy_root not in resolved_dst.parents): + raise NotInDeployRootError(dest_path=dst, deploy_root=deploy_root, src_path=src) + + absolute_src = resolve_without_follow(src) + if absolute_src.is_file(): + delete(dst) + try: + os.symlink(absolute_src, dst) + except OSError: + ssrc.copy(dst) + else: + # 1. Create a new directory in the deploy root + dst.mkdir(exist_ok=True) + # 2. For all children of src, create their counterparts in dst now that it exists + for root, _, files in sorted(os.walk(absolute_src, followlinks=True)): + relative_root = Path(root).relative_to(absolute_src) + absolute_root_in_deploy = Path(dst, relative_root) + absolute_root_in_deploy.mkdir(parents=True, exist_ok=True) + for file in sorted(files): + absolute_file_in_project = Path(absolute_src, relative_root, file) + absolute_file_in_deploy = Path(absolute_root_in_deploy, file) + symlink_or_copy( + src=absolute_file_in_project, + dst=absolute_file_in_deploy, + deploy_root=deploy_root, + ) diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 400fe726f4..5a5dfb20f1 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -4,10 +4,6 @@ import jinja2 from click import ClickException -from snowflake.cli._plugins.nativeapp.artifacts import ( - BundleMap, - resolve_without_follow, -) from snowflake.cli._plugins.nativeapp.exceptions import ( InvalidTemplateInFileError, MissingScriptError, @@ -23,6 +19,7 @@ to_stage_path, ) from snowflake.cli._plugins.stage.utils import print_diff_to_console +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console.abc import AbstractConsole from snowflake.cli.api.entities.common import get_sql_executor @@ -41,6 +38,7 @@ choose_sql_jinja_env_based_on_template_syntax, ) from snowflake.cli.api.secure_path import UNLIMITED, SecurePath +from snowflake.cli.api.utils.path_utils import resolve_without_follow from snowflake.connector import ProgrammingError diff --git a/src/snowflake/cli/api/utils/path_utils.py b/src/snowflake/cli/api/utils/path_utils.py index 2b6ae0de87..ad2c0d156f 100644 --- a/src/snowflake/cli/api/utils/path_utils.py +++ b/src/snowflake/cli/api/utils/path_utils.py @@ -14,7 +14,11 @@ from __future__ import annotations +import os import sys +from pathlib import Path + +from snowflake.cli.api.secure_path import SecurePath BUFFER_SIZE = 4096 @@ -34,3 +38,23 @@ def path_resolver(path_to_file: str) -> str: def is_stage_path(path: str) -> bool: return path.startswith("@") or path.startswith("snow://") + + +def delete(path: Path) -> None: + """ + Obliterates whatever is at the given path, or is a no-op if the + given path does not represent a file or directory that exists. + """ + spath = SecurePath(path) + if spath.path.is_file(): + spath.unlink() # remove the file + elif spath.path.is_dir(): + spath.rmdir(recursive=True) # remove dir and all contains + + +def resolve_without_follow(path: Path) -> Path: + """ + Resolves a Path to an absolute version of itself, without following + symlinks like Path.resolve() does. + """ + return Path(os.path.abspath(path)) diff --git a/tests/api/artifacts/test_bundle_map.py b/tests/api/artifacts/test_bundle_map.py new file mode 100644 index 0000000000..c74f319121 --- /dev/null +++ b/tests/api/artifacts/test_bundle_map.py @@ -0,0 +1,923 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Union + +import pytest +from snowflake.cli.api.artifacts.bundle_map import ArtifactPredicate, BundleMap +from snowflake.cli.api.artifacts.common import ( + ArtifactError, + NotInDeployRootError, + SourceNotFoundError, + TooManyFilesError, +) +from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.utils.path_utils import resolve_without_follow + +from tests.nativeapp.utils import touch +from tests.testing_utils.files_and_dirs import temp_local_dir +from tests_common import IS_WINDOWS + + +@pytest.fixture +def bundle_map(): + project_files = { + "snowflake.yml": "# empty", + "README.md": "# Test Project", + "app/setup.sql": "-- empty", + "app/manifest.yml": "# empty", + "src/snowpark/main.py": "# empty", + "src/snowpark/a/file1.py": "# empty", + "src/snowpark/a/file2.py": "# empty", + "src/snowpark/a/b/file3.py": "# empty", + "src/snowpark/a/b/file4.py": "# empty", + "src/snowpark/a/c/file5.py": "# empty", + "src/streamlit/main_ui.py": "# empty", + "src/streamlit/helpers/file1.py": "# empty", + "src/streamlit/helpers/file2.py": "# empty", + } + with temp_local_dir(project_files) as project_root: + deploy_root = project_root / "output" / "deploy" + yield BundleMap(project_root=project_root, deploy_root=deploy_root) + + +def ensure_path(path: Union[Path, str]) -> Path: + if isinstance(path, str): + return Path(path) + return path + + +def verify_mappings( + bundle_map: BundleMap, + expected_mappings: Dict[ + Union[str, Path], Optional[Union[str, Path, List[str], List[Path]]] + ], + expected_deploy_paths: ( + Dict[Union[str, Path], Optional[Union[str, Path, List[str], List[Path]]]] | None + ) = None, + **kwargs, +): + def normalize_expected_dest( + dest: Optional[Union[str, Path, List[str], List[Path]]] + ): + if dest is None: + return [] + elif isinstance(dest, str): + return [ensure_path(dest)] + elif isinstance(dest, Path): + return [dest] + else: + return sorted([ensure_path(d) for d in dest]) + + normalized_expected_mappings = { + ensure_path(src): normalize_expected_dest(dest) + for src, dest in expected_mappings.items() + if dest is not None + } + if expected_deploy_paths is not None: + normalized_expected_deploy_paths = { + ensure_path(src): normalize_expected_dest(dest) + for src, dest in expected_deploy_paths.items() + } + else: + normalized_expected_deploy_paths = normalized_expected_mappings + + for src, expected_dests in normalized_expected_deploy_paths.items(): + assert sorted(bundle_map.to_deploy_paths(ensure_path(src))) == expected_dests + + actual_path_mappings: Dict[Path, List[Path]] = {} + for src, dest in bundle_map.all_mappings(**kwargs): + mappings = actual_path_mappings.setdefault(src, []) + mappings.append(dest) + mappings.sort() + + assert actual_path_mappings == normalized_expected_mappings + + +def verify_sources( + bundle_map: BundleMap, expected_sources: Iterable[Union[str, Path]], **kwargs +) -> None: + actual_sources = sorted(bundle_map.all_sources(**kwargs)) + expected_sources = sorted([ensure_path(src) for src in expected_sources]) + assert actual_sources == expected_sources + + +def test_empty_bundle_map(bundle_map): + mappings = list(bundle_map.all_mappings()) + assert mappings == [] + + verify_sources(bundle_map, []) + + verify_mappings( + bundle_map, + { + "app/setup.sql": None, + ".": None, + "/not/in/project": None, + }, + ) + + +def test_bundle_map_requires_absolute_project_root(): + project_root = Path() + with pytest.raises( + AssertionError, + match=re.escape(rf"Project root {project_root} must be an absolute path."), + ): + BundleMap(project_root=project_root, deploy_root=Path("output/deploy")) + + +def test_bundle_map_requires_absolute_deploy_root(): + deploy_root = Path("output/deploy") + with pytest.raises( + AssertionError, + match=re.escape(rf"Deploy root {deploy_root} must be an absolute path."), + ): + BundleMap(project_root=Path().resolve(), deploy_root=deploy_root) + + +def test_bundle_map_handles_file_to_file_mappings(bundle_map): + bundle_map.add(PathMapping(src="README.md", dest="deployed_readme.md")) + bundle_map.add(PathMapping(src="app/setup.sql", dest="app_setup.sql")) + bundle_map.add(PathMapping(src="app/manifest.yml", dest="manifest.yml")) + + verify_mappings( + bundle_map, + { + "README.md": "deployed_readme.md", + "app/setup.sql": "app_setup.sql", + "app/manifest.yml": "manifest.yml", + }, + ) + + verify_sources(bundle_map, ["README.md", "app/setup.sql", "app/manifest.yml"]) + + +def test_bundle_map_supports_double_star_glob(bundle_map): + bundle_map.add(PathMapping(src="src/snowpark/**/*.py", dest="deployed/")) + + expected_mappings = { + "src/snowpark/main.py": "deployed/main.py", + "src/snowpark/a/file1.py": "deployed/file1.py", + "src/snowpark/a/file2.py": "deployed/file2.py", + "src/snowpark/a/b/file3.py": "deployed/file3.py", + "src/snowpark/a/b/file4.py": "deployed/file4.py", + "src/snowpark/a/c/file5.py": "deployed/file5.py", + } + + verify_mappings(bundle_map, expected_mappings) + + verify_sources(bundle_map, expected_mappings.keys()) + + +def test_bundle_map_supports_complex_globbing(bundle_map): + bundle_map.add(PathMapping(src="src/s*/**/file[3-5].py", dest="deployed/")) + + expected_mappings = { + "src/snowpark/main.py": None, + "src/snowpark/a/file1.py": None, + "src/snowpark/a/file2.py": None, + "src/snowpark/a/b/file3.py": "deployed/file3.py", + "src/snowpark/a/b/file4.py": "deployed/file4.py", + "src/snowpark/a/c/file5.py": "deployed/file5.py", + } + + verify_mappings( + bundle_map, + expected_mappings, + ) + + verify_sources( + bundle_map, + [src for src in expected_mappings.keys() if expected_mappings[src] is not None], + ) + + +def test_bundle_map_handles_mapping_to_deploy_root(bundle_map): + bundle_map.add(PathMapping(src="app/*", dest="./")) + bundle_map.add(PathMapping(src="README.md", dest="./")) + + verify_mappings( + bundle_map, + { + "app/setup.sql": "setup.sql", + "app/manifest.yml": "manifest.yml", + "README.md": "README.md", + }, + ) + + +def test_bundle_map_can_rename_directories(bundle_map): + bundle_map.add(PathMapping(src="app", dest="deployed")) + + verify_mappings( + bundle_map, + { + "app": "deployed", + }, + expand_directories=False, + ) + + verify_mappings( + bundle_map, + { + "app": "deployed", + "app/setup.sql": "deployed/setup.sql", + "app/manifest.yml": "deployed/manifest.yml", + }, + expand_directories=True, + ) + + +def test_bundle_map_honours_trailing_slashes(bundle_map): + bundle_map.add(PathMapping(src="app", dest="deployed/")) + bundle_map.add(PathMapping(src="README.md", dest="deployed/")) + bundle_map.add( + # src trailing slash has no effect + PathMapping(src="src/snowpark/", dest="deployed/") + ) + + verify_mappings( + bundle_map, + { + "app": "deployed/app", + "src/snowpark": "deployed/snowpark", + "README.md": "deployed/README.md", + }, + ) + + verify_mappings( + bundle_map, + { + "app": "deployed/app", + "app/manifest.yml": "deployed/app/manifest.yml", + "app/setup.sql": "deployed/app/setup.sql", + "src/snowpark": "deployed/snowpark", + "src/snowpark/main.py": "deployed/snowpark/main.py", + "src/snowpark/a": "deployed/snowpark/a", + "src/snowpark/a/file1.py": "deployed/snowpark/a/file1.py", + "src/snowpark/a/file2.py": "deployed/snowpark/a/file2.py", + "src/snowpark/a/b": "deployed/snowpark/a/b", + "src/snowpark/a/b/file3.py": "deployed/snowpark/a/b/file3.py", + "src/snowpark/a/b/file4.py": "deployed/snowpark/a/b/file4.py", + "src/snowpark/a/c": "deployed/snowpark/a/c", + "src/snowpark/a/c/file5.py": "deployed/snowpark/a/c/file5.py", + "README.md": "deployed/README.md", + }, + expand_directories=True, + ) + + +def test_bundle_map_disallows_overwriting_deploy_root(bundle_map): + with pytest.raises(NotInDeployRootError): + bundle_map.add(PathMapping(src="app/*", dest=".")) + + +def test_bundle_map_disallows_unknown_sources(bundle_map): + with pytest.raises(SourceNotFoundError): + bundle_map.add(PathMapping(src="missing/*", dest="deployed/")) + + with pytest.raises(SourceNotFoundError): + bundle_map.add(PathMapping(src="missing", dest="deployed/")) + + with pytest.raises(SourceNotFoundError): + bundle_map.add(PathMapping(src="**/*.missing", dest="deployed/")) + + +def test_bundle_map_disallows_mapping_multiple_to_file(bundle_map): + with pytest.raises(TooManyFilesError): + # multiple files named 'file1.py' would collide + bundle_map.add(PathMapping(src="**/file1.py", dest="deployed/")) + + with pytest.raises(TooManyFilesError): + bundle_map.add(PathMapping(src="**/file1.py", dest="deployed/")) + + +def test_bundle_map_allows_mapping_file_to_multiple_destinations(bundle_map): + bundle_map.add(PathMapping(src="README.md", dest="deployed/README1.md")) + bundle_map.add(PathMapping(src="README.md", dest="deployed/README2.md")) + bundle_map.add(PathMapping(src="src/streamlit", dest="deployed/streamlit_orig")) + bundle_map.add(PathMapping(src="src/streamlit", dest="deployed/streamlit_copy")) + bundle_map.add(PathMapping(src="src/streamlit/main_ui.py", dest="deployed/")) + + verify_mappings( + bundle_map, + expected_mappings={ + "README.md": ["deployed/README1.md", "deployed/README2.md"], + "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], + "src/streamlit/main_ui.py": ["deployed/main_ui.py"], + }, + expected_deploy_paths={ + "README.md": ["deployed/README1.md", "deployed/README2.md"], + "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], + "src/streamlit/main_ui.py": [ + "deployed/main_ui.py", + "deployed/streamlit_orig/main_ui.py", + "deployed/streamlit_copy/main_ui.py", + ], + }, + ) + + verify_mappings( + bundle_map, + expected_mappings={ + "README.md": ["deployed/README1.md", "deployed/README2.md"], + "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], + "src/streamlit/main_ui.py": [ + "deployed/main_ui.py", + "deployed/streamlit_orig/main_ui.py", + "deployed/streamlit_copy/main_ui.py", + ], + "src/streamlit/helpers": [ + "deployed/streamlit_orig/helpers", + "deployed/streamlit_copy/helpers", + ], + "src/streamlit/helpers/file1.py": [ + "deployed/streamlit_orig/helpers/file1.py", + "deployed/streamlit_copy/helpers/file1.py", + ], + "src/streamlit/helpers/file2.py": [ + "deployed/streamlit_orig/helpers/file2.py", + "deployed/streamlit_copy/helpers/file2.py", + ], + }, + expected_deploy_paths={ + "README.md": ["deployed/README1.md", "deployed/README2.md"], + "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], + "src/streamlit/main_ui.py": [ + "deployed/main_ui.py", + "deployed/streamlit_orig/main_ui.py", + "deployed/streamlit_copy/main_ui.py", + ], + "src/streamlit/helpers": [ + "deployed/streamlit_orig/helpers", + "deployed/streamlit_copy/helpers", + ], + "src/streamlit/helpers/file1.py": [ + "deployed/streamlit_orig/helpers/file1.py", + "deployed/streamlit_copy/helpers/file1.py", + ], + "src/streamlit/helpers/file2.py": [ + "deployed/streamlit_orig/helpers/file2.py", + "deployed/streamlit_copy/helpers/file2.py", + ], + }, + expand_directories=True, + ) + + +def test_bundle_map_handles_missing_dest(bundle_map): + bundle_map.add(PathMapping(src="app")) + bundle_map.add(PathMapping(src="README.md")) + bundle_map.add(PathMapping(src="src/streamlit/")) + + verify_mappings( + bundle_map, + {"app": "app", "README.md": "README.md", "src/streamlit": "src/streamlit"}, + ) + + verify_mappings( + bundle_map, + { + "app": "app", + "app/setup.sql": "app/setup.sql", + "app/manifest.yml": "app/manifest.yml", + "README.md": "README.md", + "src/streamlit": "src/streamlit", + "src/streamlit/helpers": "src/streamlit/helpers", + "src/streamlit/main_ui.py": "src/streamlit/main_ui.py", + "src/streamlit/helpers/file1.py": "src/streamlit/helpers/file1.py", + "src/streamlit/helpers/file2.py": "src/streamlit/helpers/file2.py", + }, + expand_directories=True, + ) + + +def test_bundle_map_disallows_mapping_files_as_directories(bundle_map): + bundle_map.add(PathMapping(src="app", dest="deployed/")) + with pytest.raises( + ArtifactError, match="Conflicting type for destination path: deployed" + ): + bundle_map.add(PathMapping(src="**/main.py", dest="deployed")) + + +def test_bundle_map_disallows_mapping_directories_as_files(bundle_map): + bundle_map.add(PathMapping(src="**/main.py", dest="deployed")) + with pytest.raises( + ArtifactError, match="Conflicting type for destination path: deployed" + ): + bundle_map.add(PathMapping(src="app", dest="deployed")) + + +def test_bundle_map_allows_deploying_other_sources_to_renamed_directory(bundle_map): + bundle_map.add(PathMapping(src="src/snowpark", dest="./snowpark")) + bundle_map.add(PathMapping(src="README.md", dest="snowpark/")) + + verify_mappings( + bundle_map, + { + "src/snowpark": "snowpark", + "README.md": "snowpark/README.md", + }, + ) + + verify_mappings( + bundle_map, + { + "README.md": "snowpark/README.md", + "src/snowpark": "snowpark", + "src/snowpark/main.py": "snowpark/main.py", + "src/snowpark/a": "snowpark/a", + "src/snowpark/a/file1.py": "snowpark/a/file1.py", + "src/snowpark/a/file2.py": "snowpark/a/file2.py", + "src/snowpark/a/b": "snowpark/a/b", + "src/snowpark/a/b/file3.py": "snowpark/a/b/file3.py", + "src/snowpark/a/b/file4.py": "snowpark/a/b/file4.py", + "src/snowpark/a/c": "snowpark/a/c", + "src/snowpark/a/c/file5.py": "snowpark/a/c/file5.py", + }, + expand_directories=True, + ) + + +@pytest.mark.skip(reason="Checking deep tree hierarchies is not yet supported") +def test_bundle_map_disallows_collisions_anywhere_in_deployed_hierarchy(bundle_map): + bundle_map.add(PathMapping(src="src/snowpark", dest="./snowpark")) + bundle_map.add(PathMapping(src="README.md", dest="snowpark/")) + + # if any of the files collide, however, this is not allowed + with pytest.raises(TooManyFilesError): + bundle_map.add(PathMapping(src="app/manifest.yml", dest="snowpark/README.md")) + + with pytest.raises(TooManyFilesError): + bundle_map.add(PathMapping(src="app/manifest.yml", dest="snowpark/a/file1.py")) + + +def test_bundle_map_disallows_mapping_outside_deploy_root(bundle_map): + with pytest.raises(NotInDeployRootError): + bundle_map.add(PathMapping(src="app", dest="deployed/../../")) + + with pytest.raises(NotInDeployRootError): + bundle_map.add(PathMapping(src="app", dest=Path().resolve().root)) + + with pytest.raises(NotInDeployRootError): + bundle_map.add(PathMapping(src="app", dest="/////")) + + +def test_bundle_map_disallows_absolute_src(bundle_map): + with pytest.raises(ArtifactError): + absolute_src = bundle_map.project_root() / "app" + assert absolute_src.is_absolute() + bundle_map.add(PathMapping(src=str(absolute_src), dest="deployed")) + + +def test_bundle_map_disallows_absolute_dest(bundle_map): + with pytest.raises(ArtifactError): + absolute_dest = bundle_map.deploy_root() / "deployed" + assert absolute_dest.is_absolute() + bundle_map.add(PathMapping(src="app", dest=str(absolute_dest))) + + +def test_bundle_map_disallows_clobbering_parent_directories(bundle_map): + # one level of nesting + with pytest.raises(TooManyFilesError): + bundle_map.add(PathMapping(src="snowflake.yml", dest="./app/")) + # Adding a new rule to populate ./app/ from an existing directory. This would + # clobber the output of the previous rule, so it's disallowed + bundle_map.add(PathMapping(src="./app", dest="./")) + + # same as above but with multiple levels of nesting + with pytest.raises(TooManyFilesError): + bundle_map.add(PathMapping(src="snowflake.yml", dest="./src/snowpark/a/")) + bundle_map.add(PathMapping(src="./src/snowpark", dest="./src/")) + + +def test_bundle_map_disallows_clobbering_child_directories(bundle_map): + with pytest.raises(TooManyFilesError): + bundle_map.add(PathMapping(src="./src/snowpark", dest="./python/")) + bundle_map.add(PathMapping(src="./app", dest="./python/snowpark/a")) + + +def test_bundle_map_allows_augmenting_dest_directories(bundle_map): + # one level of nesting + # First populate {deploy}/app from an existing directory + bundle_map.add(PathMapping(src="./app", dest="./")) + # Then add a new file to that directory + bundle_map.add(PathMapping(src="snowflake.yml", dest="./app/")) + + # verify that when iterating over mappings, the base directory rule appears first, + # followed by the file. This is important for correctness, and should be + # deterministic + ordered_dests = [ + dest for (_, dest) in bundle_map.all_mappings(expand_directories=True) + ] + file_index = ordered_dests.index(Path("app/snowflake.yml")) + dir_index = ordered_dests.index(Path("app")) + assert dir_index < file_index + + +def test_bundle_map_allows_augmenting_dest_directories_nested(bundle_map): + # same as above but with multiple levels of nesting + bundle_map.add(PathMapping(src="./src/snowpark", dest="./src/")) + bundle_map.add(PathMapping(src="snowflake.yml", dest="./src/snowpark/a/")) + + ordered_dests = [ + dest for (_, dest) in bundle_map.all_mappings(expand_directories=True) + ] + file_index = ordered_dests.index(Path("src/snowpark/a/snowflake.yml")) + dir_index = ordered_dests.index(Path("src/snowpark")) + assert dir_index < file_index + + +def test_bundle_map_returns_mappings_in_insertion_order(bundle_map): + # this behaviour is important to make sure the deploy root is populated in a + # deterministic manner, so verify it here + bundle_map.add(PathMapping(src="./app", dest="./")) + bundle_map.add(PathMapping(src="snowflake.yml", dest="./app/")) + bundle_map.add(PathMapping(src="./src/snowpark", dest="./src/")) + bundle_map.add(PathMapping(src="snowflake.yml", dest="./src/snowpark/a/")) + + ordered_dests = [ + dest for (_, dest) in bundle_map.all_mappings(expand_directories=False) + ] + assert ordered_dests == [ + Path("app"), + Path("app/snowflake.yml"), + Path("src/snowpark"), + Path("src/snowpark/a/snowflake.yml"), + ] + + +def test_bundle_map_all_mappings_generates_absolute_directories_when_requested( + bundle_map, +): + project_root = bundle_map.project_root() + assert project_root.is_absolute() + deploy_root = bundle_map.deploy_root() + assert deploy_root.is_absolute() + + bundle_map.add(PathMapping(src="app", dest="deployed_app")) + bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) + bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) + + verify_mappings( + bundle_map, + { + "app": "deployed_app", + "README.md": "deployed_README.md", + "src/streamlit": "deployed_streamlit", + }, + ) + + verify_mappings( + bundle_map, + { + project_root / "app": deploy_root / "deployed_app", + project_root / "README.md": deploy_root / "deployed_README.md", + project_root / "src/streamlit": deploy_root / "deployed_streamlit", + }, + absolute=True, + expand_directories=False, + ) + + verify_mappings( + bundle_map, + { + project_root / "app": deploy_root / "deployed_app", + project_root / "app/setup.sql": deploy_root / "deployed_app/setup.sql", + project_root + / "app/manifest.yml": deploy_root + / "deployed_app/manifest.yml", + project_root / "README.md": deploy_root / "deployed_README.md", + project_root / "src/streamlit": deploy_root / "deployed_streamlit", + project_root + / "src/streamlit/helpers": deploy_root + / "deployed_streamlit/helpers", + project_root + / "src/streamlit/main_ui.py": deploy_root + / "deployed_streamlit/main_ui.py", + project_root + / "src/streamlit/helpers/file1.py": deploy_root + / "deployed_streamlit/helpers/file1.py", + project_root + / "src/streamlit/helpers/file2.py": deploy_root + / "deployed_streamlit/helpers/file2.py", + }, + absolute=True, + expand_directories=True, + ) + + +def test_bundle_map_all_sources_generates_absolute_directories_when_requested( + bundle_map, +): + project_root = bundle_map.project_root() + assert project_root.is_absolute() + + bundle_map.add(PathMapping(src="app", dest="deployed_app")) + bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) + bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) + + verify_sources(bundle_map, ["app", "README.md", "src/streamlit"]) + + verify_sources( + bundle_map, + [ + project_root / "app", + project_root / "README.md", + project_root / "src/streamlit", + ], + absolute=True, + ) + + +def test_bundle_map_all_mappings_accepts_predicates(bundle_map): + project_root = bundle_map.project_root() + assert project_root.is_absolute() + deploy_root = bundle_map.deploy_root() + assert deploy_root.is_absolute() + + bundle_map.add(PathMapping(src="app", dest="deployed_app")) + bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) + bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) + + collected: Dict[Path, Path] = {} + + def collecting_predicate(predicate: ArtifactPredicate) -> ArtifactPredicate: + def _predicate(src: Path, dest: Path) -> bool: + collected[src] = dest + return predicate(src, dest) + + return _predicate + + verify_mappings( + bundle_map, + { + project_root + / "src/streamlit/main_ui.py": deploy_root + / "deployed_streamlit/main_ui.py", + project_root + / "src/streamlit/helpers/file1.py": deploy_root + / "deployed_streamlit/helpers/file1.py", + project_root + / "src/streamlit/helpers/file2.py": deploy_root + / "deployed_streamlit/helpers/file2.py", + }, + absolute=True, + expand_directories=True, + predicate=collecting_predicate( + lambda src, dest: src.is_file() and src.suffix == ".py" + ), + ) + + assert collected == { + project_root / "app": deploy_root / "deployed_app", + project_root / "app/setup.sql": deploy_root / "deployed_app/setup.sql", + project_root / "app/manifest.yml": deploy_root / "deployed_app/manifest.yml", + project_root / "README.md": deploy_root / "deployed_README.md", + project_root / "src/streamlit": deploy_root / "deployed_streamlit", + project_root + / "src/streamlit/helpers": deploy_root + / "deployed_streamlit/helpers", + project_root + / "src/streamlit/main_ui.py": deploy_root + / "deployed_streamlit/main_ui.py", + project_root + / "src/streamlit/helpers/file1.py": deploy_root + / "deployed_streamlit/helpers/file1.py", + project_root + / "src/streamlit/helpers/file2.py": deploy_root + / "deployed_streamlit/helpers/file2.py", + } + + collected = {} + + verify_mappings( + bundle_map, + { + "src/streamlit/main_ui.py": "deployed_streamlit/main_ui.py", + "src/streamlit/helpers/file1.py": "deployed_streamlit/helpers/file1.py", + "src/streamlit/helpers/file2.py": "deployed_streamlit/helpers/file2.py", + }, + absolute=False, + expand_directories=True, + predicate=collecting_predicate(lambda src, dest: src.suffix == ".py"), + ) + + assert collected == { + Path("app"): Path("deployed_app"), + Path("app/setup.sql"): Path("deployed_app/setup.sql"), + Path("app/manifest.yml"): Path("deployed_app/manifest.yml"), + Path("README.md"): Path("deployed_README.md"), + Path("src/streamlit"): Path("deployed_streamlit"), + Path("src/streamlit/main_ui.py"): Path("deployed_streamlit/main_ui.py"), + Path("src/streamlit/helpers"): Path("deployed_streamlit/helpers"), + Path("src/streamlit/helpers/file1.py"): Path( + "deployed_streamlit/helpers/file1.py" + ), + Path("src/streamlit/helpers/file2.py"): Path( + "deployed_streamlit/helpers/file2.py" + ), + } + + +def test_bundle_map_to_deploy_path(bundle_map): + bundle_map.add(PathMapping(src="app", dest="deployed_app")) + bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) + bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) + + # to_deploy_path returns relative paths when relative paths are given as input + assert bundle_map.to_deploy_paths(Path("app")) == [Path("deployed_app")] + assert bundle_map.to_deploy_paths(Path("README.md")) == [Path("deployed_README.md")] + assert bundle_map.to_deploy_paths(Path("src/streamlit")) == [ + Path("deployed_streamlit") + ] + assert bundle_map.to_deploy_paths(Path("src/streamlit/main_ui.py")) == [ + Path("deployed_streamlit/main_ui.py") + ] + assert bundle_map.to_deploy_paths(Path("src/streamlit/helpers")) == [ + Path("deployed_streamlit/helpers") + ] + assert bundle_map.to_deploy_paths(Path("src/streamlit/helpers/file1.py")) == [ + Path("deployed_streamlit/helpers/file1.py") + ] + assert bundle_map.to_deploy_paths(Path("src/streamlit/missing.py")) == [] + assert bundle_map.to_deploy_paths(Path("missing")) == [] + assert bundle_map.to_deploy_paths(Path("src/missing/")) == [] + assert bundle_map.to_deploy_paths(bundle_map.project_root().parent) == [] + + # to_deploy_path returns absolute paths when absolute paths are given as input + project_root = bundle_map.project_root() + deploy_root = bundle_map.deploy_root() + assert bundle_map.to_deploy_paths(project_root / "app") == [ + deploy_root / "deployed_app" + ] + assert bundle_map.to_deploy_paths(project_root / "README.md") == [ + deploy_root / "deployed_README.md" + ] + assert bundle_map.to_deploy_paths(project_root / "src/streamlit") == [ + deploy_root / "deployed_streamlit" + ] + assert bundle_map.to_deploy_paths(project_root / "src/streamlit/main_ui.py") == [ + deploy_root / "deployed_streamlit/main_ui.py" + ] + assert bundle_map.to_deploy_paths(project_root / "src/streamlit/helpers") == [ + deploy_root / "deployed_streamlit/helpers" + ] + assert bundle_map.to_deploy_paths( + project_root / "src/streamlit/helpers/file1.py" + ) == [deploy_root / "deployed_streamlit/helpers/file1.py"] + assert bundle_map.to_deploy_paths(project_root / "src/streamlit/missing.py") == [] + + +def test_bundle_map_to_deploy_path_returns_multiple_matches(bundle_map): + bundle_map.add(PathMapping(src="src/snowpark", dest="d1")) + bundle_map.add(PathMapping(src="src/snowpark", dest="d2")) + + assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark"))) == [ + Path("d1"), + Path("d2"), + ] + + assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark/main.py"))) == [ + Path("d1/main.py"), + Path("d2/main.py"), + ] + + assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark/a/b"))) == [ + Path("d1/a/b"), + Path("d2/a/b"), + ] + + bundle_map.add(PathMapping(src="src/snowpark/a", dest="d3")) + + assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark/a/b/file3.py"))) == [ + Path("d1/a/b/file3.py"), + Path("d2/a/b/file3.py"), + Path("d3/b/file3.py"), + ] + + +@pytest.mark.parametrize( + "dest, src", + [ + ["manifest.yml", "app/manifest.yml"], + [".", None], + ["python/snowpark/main.py", "src/snowpark/main.py"], + ["python/snowpark", "src/snowpark"], + ["python/snowpark/a/b", "src/snowpark/a/b"], + ["python/snowpark/a/b/fake.py", None], + [ + # even though a rule creates this directory, it has no equivalent source folder + "python", + None, + ], + ["/fake/foo.py", None], + ], +) +def test_to_project_path(bundle_map, dest, src): + bundle_map.add(PathMapping(src="app/*", dest="./")) + bundle_map.add(PathMapping(src="src/snowpark", dest="./python/snowpark")) + + # relative paths + if src is None: + assert bundle_map.to_project_path(Path(dest)) is None + assert bundle_map.to_project_path(Path(bundle_map.deploy_root() / dest)) is None + else: + assert bundle_map.to_project_path(Path(dest)) == Path(src) + assert ( + bundle_map.to_project_path(Path(bundle_map.deploy_root() / dest)) + == bundle_map.project_root() / src + ) + + +def test_bundle_map_ignores_sources_in_deploy_root(bundle_map): + bundle_map.deploy_root().mkdir(parents=True, exist_ok=True) + deploy_root_source = bundle_map.deploy_root() / "should_not_match.yml" + touch(str(deploy_root_source)) + + bundle_map.add(PathMapping(src="**/*.yml", dest="deployed/")) + + verify_mappings( + bundle_map, + { + "app/manifest.yml": "deployed/manifest.yml", + "snowflake.yml": "deployed/snowflake.yml", + }, + ) + + +@pytest.mark.skipif( + IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" +) +@pytest.mark.parametrize( + "project_path,expected_path", + [ + [ + "srcfile", + "deploy/file", + ], + [ + "srcdir", + "deploy/dir", + ], + [ + "srcdir/nested_file1", + "deploy/dir/nested_file1", + ], + [ + "srcdir/nested_dir/nested_file2", + "deploy/dir/nested_dir/nested_file2", + ], + [ + "srcdir/nested_dir", + "deploy/dir/nested_dir", + ], + [ + "not-in-deploy", + None, + ], + ], +) +def test_source_path_to_deploy_path( + temp_dir, + project_path, + expected_path, +): + # Source files + touch("srcfile") + touch("srcdir/nested_file1") + touch("srcdir/nested_dir/nested_file2") + touch("not-in-deploy") + # Build + os.mkdir("deploy") + os.symlink("srcfile", "deploy/file") + os.symlink(Path("srcdir").resolve(), Path("deploy/dir")) + + bundle_map = BundleMap( + project_root=Path().resolve(), deploy_root=Path("deploy").resolve() + ) + bundle_map.add(PathMapping(src="srcdir", dest="./dir")) + bundle_map.add(PathMapping(src="srcfile", dest="./file")) + + result = bundle_map.to_deploy_paths(resolve_without_follow(Path(project_path))) + if expected_path: + assert result == [resolve_without_follow(Path(expected_path))] + else: + assert result == [] diff --git a/tests/api/utils/__snapshots__/test_path_utils.ambr b/tests/api/utils/__snapshots__/test_path_utils.ambr new file mode 100644 index 0000000000..837da1ec41 --- /dev/null +++ b/tests/api/utils/__snapshots__/test_path_utils.ambr @@ -0,0 +1,326 @@ +# serializer version: 1 +# name: test_symlink_or_copy_raises_error + 'Test 1' +# --- +# name: test_symlink_or_copy_raises_error.1 + 'Test 1' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root + ''' + d . + d GrandA + d GrandA/ParentA + d GrandA/ParentA/ChildA + f GrandA/ParentA/ChildA/GrandChildA + f GrandA/ParentA/ChildA/GrandChildB.py + d GrandA/ParentA/ChildA/GrandChildC + f GrandA/ParentA/ChildB.py + f GrandA/ParentA/ChildC + d GrandA/ParentA/ChildD + d GrandA/ParentB + f GrandA/ParentB/ChildA + f GrandA/ParentB/ChildB.py + d GrandA/ParentB/ChildC + d GrandA/ParentB/ChildC/GrandChildA + d GrandA/ParentC + d GrandB + d GrandB/ParentA + f GrandB/ParentA/ChildA + d output + d output/deploy + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.1 + ''' + ===== Contents of: GrandA/ParentA/ChildA/GrandChildA ===== + Text GrandA/ParentA/ChildA/GrandChildA + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.10 + ''' + ===== Contents of: output/deploy/ChildA/GrandChildB.py ===== + Text GrandA/ParentA/ChildA/GrandChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.11 + ''' + ===== Contents of: output/deploy/ChildB.py ===== + Text GrandA/ParentA/ChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.12 + ''' + ===== Contents of: output/deploy/ChildC ===== + Text GrandA/ParentA/ChildC + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.13 + ''' + ===== Contents of: output/deploy/Grand1/Parent1/Child1 ===== + Text GrandB/ParentA/ChildA + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.14 + ''' + ===== Contents of: output/deploy/Grand3 ===== + Text GrandA/ParentB/ChildA + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.15 + ''' + ===== Contents of: output/deploy/Grand4/Parent1.py ===== + Text GrandA/ParentB/ChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.16 + ''' + ===== Contents of: output/deploy/Grand4/Parent3/ChildA/GrandChildA ===== + Text GrandA/ParentA/ChildA/GrandChildA + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.17 + ''' + ===== Contents of: output/deploy/Grand4/Parent3/ChildA/GrandChildB.py ===== + Text GrandA/ParentA/ChildA/GrandChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.18 + ''' + ===== Contents of: output/deploy/Grand4/Parent3/ChildB.py ===== + Text GrandA/ParentA/ChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.19 + ''' + ===== Contents of: output/deploy/Grand4/Parent3/ChildC ===== + Text GrandA/ParentA/ChildC + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.2 + ''' + ===== Contents of: GrandA/ParentA/ChildA/GrandChildB.py ===== + Text GrandA/ParentA/ChildA/GrandChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.3 + ''' + ===== Contents of: GrandA/ParentA/ChildB.py ===== + Text GrandA/ParentA/ChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.4 + ''' + ===== Contents of: GrandA/ParentA/ChildC ===== + Text GrandA/ParentA/ChildC + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.5 + ''' + ===== Contents of: GrandA/ParentB/ChildA ===== + Text GrandA/ParentB/ChildA + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.6 + ''' + ===== Contents of: GrandA/ParentB/ChildB.py ===== + Text GrandA/ParentB/ChildB.py + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.7 + ''' + ===== Contents of: GrandB/ParentA/ChildA ===== + Text GrandB/ParentA/ChildA + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.8 + ''' + d output/deploy + d output/deploy/ChildA + f output/deploy/ChildA/GrandChildA + f output/deploy/ChildA/GrandChildB.py + d output/deploy/ChildA/GrandChildC + f output/deploy/ChildB.py + f output/deploy/ChildC + d output/deploy/ChildD + d output/deploy/Grand1 + d output/deploy/Grand1/Parent1 + f output/deploy/Grand1/Parent1/Child1 + d output/deploy/Grand2 + f output/deploy/Grand3 + d output/deploy/Grand4 + f output/deploy/Grand4/Parent1.py + d output/deploy/Grand4/Parent2 + d output/deploy/Grand4/Parent2/GrandChildA + d output/deploy/Grand4/Parent3 + d output/deploy/Grand4/Parent3/ChildA + f output/deploy/Grand4/Parent3/ChildA/GrandChildA + f output/deploy/Grand4/Parent3/ChildA/GrandChildB.py + d output/deploy/Grand4/Parent3/ChildA/GrandChildC + f output/deploy/Grand4/Parent3/ChildB.py + f output/deploy/Grand4/Parent3/ChildC + d output/deploy/Grand4/Parent3/ChildD + ''' +# --- +# name: test_symlink_or_copy_with_no_symlinks_in_project_root.9 + ''' + ===== Contents of: output/deploy/ChildA/GrandChildA ===== + Text GrandA/ParentA/ChildA/GrandChildA + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root + ''' + d . + d GrandA + f GrandA/ParentA + f GrandA/ParentB + d GrandA/ParentC + d GrandA/ParentC/ChildA + f GrandA/ParentC/ChildA/GrandChildA + f GrandA/ParentC/ChildA/GrandChildB + d GrandB + d GrandB/ParentA + d GrandB/ParentA/ChildA + f GrandB/ParentA/ChildA/GrandChildA + d GrandB/ParentA/ChildB + d GrandB/ParentA/ChildB/GrandChildA + d output + d output/deploy + d symlinks + d symlinks/Grand1 + d symlinks/Grand1/Parent3 + d symlinks/Grand1/Parent3/Child1 + d symlinks/Grand2 + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.1 + ''' + ===== Contents of: GrandA/ParentA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.10 + ''' + ===== Contents of: output/deploy/TestA/ParentA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.11 + ''' + ===== Contents of: output/deploy/TestA/ParentB ===== + Use as src of a symlink: GrandA/ParentB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.12 + ''' + ===== Contents of: output/deploy/TestA/ParentC/ChildA/GrandChildA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.13 + ''' + ===== Contents of: output/deploy/TestA/ParentC/ChildA/GrandChildB ===== + Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.14 + ''' + ===== Contents of: output/deploy/TestB/ParentA/ChildA/GrandChildA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.15 + ''' + ===== Contents of: output/deploy/symlinks/Grand1/Parent2 ===== + Use as src of a symlink: GrandA/ParentB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.16 + ''' + ===== Contents of: output/deploy/symlinks/Grand1/Parent3/Child1/GrandChild2 ===== + Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.17 + ''' + ===== Contents of: output/deploy/symlinks/Grand2/Parent1/ChildA/GrandChildA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.2 + ''' + ===== Contents of: GrandA/ParentB ===== + Use as src of a symlink: GrandA/ParentB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.3 + ''' + ===== Contents of: GrandA/ParentC/ChildA/GrandChildA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.4 + ''' + ===== Contents of: GrandA/ParentC/ChildA/GrandChildB ===== + Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.5 + ''' + ===== Contents of: GrandB/ParentA/ChildA/GrandChildA ===== + Do not use as src of a symlink + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.6 + ''' + d symlinks + d symlinks/Grand1 + f symlinks/Grand1/Parent2 + d symlinks/Grand1/Parent3 + d symlinks/Grand1/Parent3/Child1 + f symlinks/Grand1/Parent3/Child1/GrandChild2 + d symlinks/Grand2 + d symlinks/Grand2/Parent1 + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.7 + ''' + ===== Contents of: symlinks/Grand1/Parent2 ===== + Use as src of a symlink: GrandA/ParentB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.8 + ''' + ===== Contents of: symlinks/Grand1/Parent3/Child1/GrandChild2 ===== + Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB + ''' +# --- +# name: test_symlink_or_copy_with_symlinks_in_project_root.9 + ''' + d output/deploy + d output/deploy/TestA + f output/deploy/TestA/ParentA + f output/deploy/TestA/ParentB + d output/deploy/TestA/ParentC + d output/deploy/TestA/ParentC/ChildA + f output/deploy/TestA/ParentC/ChildA/GrandChildA + f output/deploy/TestA/ParentC/ChildA/GrandChildB + d output/deploy/TestB + d output/deploy/TestB/ParentA + d output/deploy/TestB/ParentA/ChildA + f output/deploy/TestB/ParentA/ChildA/GrandChildA + d output/deploy/TestB/ParentA/ChildB + d output/deploy/TestB/ParentA/ChildB/GrandChildA + d output/deploy/symlinks + d output/deploy/symlinks/Grand1 + f output/deploy/symlinks/Grand1/Parent2 + d output/deploy/symlinks/Grand1/Parent3 + d output/deploy/symlinks/Grand1/Parent3/Child1 + f output/deploy/symlinks/Grand1/Parent3/Child1/GrandChild2 + d output/deploy/symlinks/Grand2 + d output/deploy/symlinks/Grand2/Parent1 + d output/deploy/symlinks/Grand2/Parent1/ChildA + f output/deploy/symlinks/Grand2/Parent1/ChildA/GrandChildA + d output/deploy/symlinks/Grand2/Parent1/ChildB + d output/deploy/symlinks/Grand2/Parent1/ChildB/GrandChildA + ''' +# --- diff --git a/tests/api/utils/test_path_utils.py b/tests/api/utils/test_path_utils.py new file mode 100644 index 0000000000..67d9ddd222 --- /dev/null +++ b/tests/api/utils/test_path_utils.py @@ -0,0 +1,340 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from snowflake.cli.api.artifacts.common import NotInDeployRootError +from snowflake.cli.api.artifacts.utils import symlink_or_copy + +from tests.nativeapp.utils import assert_dir_snapshot, touch +from tests.testing_utils.files_and_dirs import pushd, temp_local_dir +from tests_common import IS_WINDOWS + + +@pytest.mark.skipif( + IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" +) +def test_symlink_or_copy_raises_error(temp_dir, os_agnostic_snapshot): + touch("GrandA/ParentA/ChildA") + with open(Path(temp_dir, "GrandA/ParentA/ChildA"), "w") as f: + f.write("Test 1") + + # Create the deploy root + deploy_root = Path(temp_dir, "output", "deploy") + os.makedirs(deploy_root) + + # Incorrect dst path + with pytest.raises(NotInDeployRootError): + symlink_or_copy( + src=Path("GrandA", "ParentA", "ChildA"), + dst=Path("output", "ParentA", "ChildA"), + deploy_root=deploy_root, + ) + + file_in_deploy_root = Path("output", "deploy", "ParentA", "ChildA") + + # Correct path and parent directories are automatically created + symlink_or_copy( + src=Path("GrandA", "ParentA", "ChildA"), + dst=file_in_deploy_root, + deploy_root=deploy_root, + ) + + assert file_in_deploy_root.exists() and file_in_deploy_root.is_symlink() + assert file_in_deploy_root.read_text(encoding="utf-8") == os_agnostic_snapshot + + # Since file_in_deploy_root is a symlink + # it resolves to project_dir/GrandA/ParentA/ChildA, which is not in deploy root + with pytest.raises(NotInDeployRootError): + symlink_or_copy( + src=Path("GrandA", "ParentA", "ChildA"), + dst=file_in_deploy_root, + deploy_root=deploy_root, + ) + + # Unlink the symlink file and create a file with the same name and path + # This should pass since src.is_file() always begins by deleting the dst. + os.unlink(file_in_deploy_root) + touch(file_in_deploy_root) + symlink_or_copy( + src=Path("GrandA", "ParentA", "ChildA"), + dst=file_in_deploy_root, + deploy_root=deploy_root, + ) + + # dst is an existing symlink, will resolve to the src during NotInDeployRootError check. + touch("GrandA/ParentA/ChildB") + with pytest.raises(NotInDeployRootError): + symlink_or_copy( + src=Path("GrandA/ParentA/ChildB"), + dst=file_in_deploy_root, + deploy_root=deploy_root, + ) + assert file_in_deploy_root.exists() and file_in_deploy_root.is_symlink() + assert file_in_deploy_root.read_text(encoding="utf-8") == os_agnostic_snapshot + + +@pytest.mark.skipif( + IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" +) +def test_symlink_or_copy_with_no_symlinks_in_project_root(os_agnostic_snapshot): + test_dir_structure = { + "GrandA/ParentA/ChildA/GrandChildA": "Text GrandA/ParentA/ChildA/GrandChildA", + "GrandA/ParentA/ChildA/GrandChildB.py": "Text GrandA/ParentA/ChildA/GrandChildB.py", + "GrandA/ParentA/ChildA/GrandChildC": None, # dir + "GrandA/ParentA/ChildB.py": "Text GrandA/ParentA/ChildB.py", + "GrandA/ParentA/ChildC": "Text GrandA/ParentA/ChildC", + "GrandA/ParentA/ChildD": None, # dir + "GrandA/ParentB/ChildA": "Text GrandA/ParentB/ChildA", + "GrandA/ParentB/ChildB.py": "Text GrandA/ParentB/ChildB.py", + "GrandA/ParentB/ChildC/GrandChildA": None, # dir + "GrandA/ParentC": None, # dir + "GrandB/ParentA/ChildA": "Text GrandB/ParentA/ChildA", + "output/deploy": None, # dir + } + with temp_local_dir(test_dir_structure) as project_root: + with pushd(project_root): + # Sanity Check + assert_dir_snapshot(Path("."), os_agnostic_snapshot) + + deploy_root = Path(project_root, "output/deploy") + + # "GrandB" dir + symlink_or_copy( + src=Path("GrandB/ParentA/ChildA"), + dst=Path(deploy_root, "Grand1/Parent1/Child1"), + deploy_root=deploy_root, + ) + assert not Path(deploy_root, "Grand1").is_symlink() + assert not Path(deploy_root, "Grand1/Parent1").is_symlink() + assert Path(deploy_root, "Grand1/Parent1/Child1").is_symlink() + + # "GrandA/ParentC" dir + symlink_or_copy( + src=Path("GrandA/ParentC"), + dst=Path(deploy_root, "Grand2"), + deploy_root=deploy_root, + ) + assert not Path(deploy_root, "Grand2").is_symlink() + + # "GrandA/ParentB" dir + symlink_or_copy( + src=Path("GrandA/ParentB/ChildA"), + dst=Path(deploy_root, "Grand3"), + deploy_root=deploy_root, + ) + assert Path(deploy_root, "Grand3").is_symlink() + symlink_or_copy( + src=Path("GrandA/ParentB/ChildB.py"), + dst=Path(deploy_root, "Grand4/Parent1.py"), + deploy_root=deploy_root, + ) + assert not Path(deploy_root, "Grand4").is_symlink() + assert Path(deploy_root, "Grand4/Parent1.py").is_symlink() + symlink_or_copy( + src=Path("GrandA/ParentB/ChildC"), + dst=Path(deploy_root, "Grand4/Parent2"), + deploy_root=deploy_root, + ) + assert not Path(deploy_root, "Grand4").is_symlink() + assert not Path(deploy_root, "Grand4/Parent2").is_symlink() + assert not Path(deploy_root, "Grand4/Parent2/GrandChildA").is_symlink() + + # "GrandA/ParentA" dir (1) + symlink_or_copy( + src=Path("GrandA/ParentA"), dst=deploy_root, deploy_root=deploy_root + ) + assert not deploy_root.is_symlink() + assert not Path(deploy_root, "ChildA").is_symlink() + assert Path(deploy_root, "ChildA/GrandChildA").is_symlink() + assert Path(deploy_root, "ChildA/GrandChildB.py").is_symlink() + assert not Path(deploy_root, "ChildA/GrandChildC").is_symlink() + assert Path(deploy_root, "ChildB.py").is_symlink() + assert Path(deploy_root, "ChildC").is_symlink() + assert not Path(deploy_root, "ChildD").is_symlink() + + # "GrandA/ParentA" dir (2) + symlink_or_copy( + src=Path("GrandA/ParentA"), + dst=Path(deploy_root, "Grand4/Parent3"), + deploy_root=deploy_root, + ) + # Other children of Grand4 will be verified by a full assert_dir_snapshot(project_root) below + assert not Path(deploy_root, "Grand4/Parent3").is_symlink() + assert not Path(deploy_root, "Grand4/Parent3/ChildA").is_symlink() + assert Path(deploy_root, "Grand4/Parent3/ChildA/GrandChildA").is_symlink() + assert Path( + deploy_root, "Grand4/Parent3/ChildA/GrandChildB.py" + ).is_symlink() + assert not Path( + deploy_root, "Grand4/Parent3/ChildA/GrandChildC" + ).is_symlink() + assert Path(deploy_root, "Grand4/Parent3/ChildB.py").is_symlink() + assert Path(deploy_root, "Grand4/Parent3/ChildC").is_symlink() + assert not Path(deploy_root, "Grand4/Parent3/ChildD").is_symlink() + + assert_dir_snapshot(Path("./output/deploy"), os_agnostic_snapshot) + + # This is because the dst can be symlinks, which resolves to project src and hence outside deploy root. + with pytest.raises(NotInDeployRootError): + symlink_or_copy( + src=Path("GrandA/ParentB"), + dst=Path(deploy_root, "Grand4/Parent3"), + deploy_root=deploy_root, + ) + + +@pytest.mark.skipif( + IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" +) +def test_symlink_or_copy_with_symlinks_in_project_root(os_agnostic_snapshot): + test_dir_structure = { + "GrandA/ParentA": "Do not use as src of a symlink", + "GrandA/ParentB": "Use as src of a symlink: GrandA/ParentB", + "GrandA/ParentC/ChildA/GrandChildA": "Do not use as src of a symlink", + "GrandA/ParentC/ChildA/GrandChildB": "Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB", + "GrandB/ParentA/ChildA/GrandChildA": "Do not use as src of a symlink", + "GrandB/ParentA/ChildB/GrandChildA": None, + "symlinks/Grand1/Parent3/Child1": None, + "symlinks/Grand2": None, + "output/deploy": None, # dir + } + with temp_local_dir(test_dir_structure) as project_root: + with pushd(project_root): + # Sanity Check + assert_dir_snapshot(Path("."), os_agnostic_snapshot) + + os.symlink( + Path("GrandA/ParentB").resolve(), + Path(project_root, "symlinks/Grand1/Parent2"), + ) + os.symlink( + Path("GrandA/ParentC/ChildA/GrandChildB").resolve(), + Path(project_root, "symlinks/Grand1/Parent3/Child1/GrandChild2"), + ) + os.symlink( + Path("GrandB/ParentA").resolve(), + Path(project_root, "symlinks/Grand2/Parent1"), + target_is_directory=True, + ) + assert Path("symlinks").is_dir() and not Path("symlinks").is_symlink() + assert ( + Path("GrandA/ParentB").is_file() + and not Path("GrandA/ParentB").is_symlink() + ) + assert ( + Path("symlinks/Grand1/Parent2").is_symlink() + and Path("symlinks/Grand1/Parent2").is_file() + ) + assert ( + Path("symlinks/Grand1/Parent3/Child1/GrandChild2").is_symlink() + and Path("symlinks/Grand1/Parent3/Child1/GrandChild2").is_file() + ) + assert ( + Path("symlinks/Grand2/Parent1").is_symlink() + and Path("symlinks/Grand2/Parent1").is_dir() + ) + + # Sanity Check + assert_dir_snapshot(Path("./symlinks"), os_agnostic_snapshot) + + deploy_root = Path(project_root, "output/deploy") + + symlink_or_copy( + src=Path("GrandA"), + dst=Path(deploy_root, "TestA"), + deploy_root=deploy_root, + ) + assert not Path(deploy_root, "TestA").is_symlink() + assert Path(deploy_root, "TestA/ParentA").is_symlink() + assert Path(deploy_root, "TestA/ParentB").is_symlink() + assert not Path(deploy_root, "TestA/ParentC").is_symlink() + assert not Path(deploy_root, "TestA/ParentC/ChildA").is_symlink() + assert Path(deploy_root, "TestA/ParentC/ChildA/GrandChildA").is_symlink() + assert Path(deploy_root, "TestA/ParentC/ChildA/GrandChildB").is_symlink() + + symlink_or_copy( + src=Path("GrandB"), + dst=Path(deploy_root, "TestB"), + deploy_root=deploy_root, + ) + assert not Path(deploy_root, "TestB").is_symlink() + assert not Path(deploy_root, "TestB/ParentA").is_symlink() + assert not Path(deploy_root, "TestB/ParentA/ChildA").is_symlink() + assert not Path(deploy_root, "TestB/ParentA/ChildB").is_symlink() + assert not Path( + deploy_root, "TestB/ParentA/ChildB/GrandChildA" + ).is_symlink() + assert Path(deploy_root, "TestB/ParentA/ChildA/GrandChildA").is_symlink() + + symlink_or_copy( + src=Path("symlinks"), + dst=Path(deploy_root, "symlinks"), + deploy_root=deploy_root, + ) + assert ( + Path(deploy_root, "symlinks/Grand1").is_dir() + and not Path(deploy_root, "symlinks/Grand1").is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand1/Parent2").is_file() + and Path(deploy_root, "symlinks/Grand1/Parent2").is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand1/Parent3").is_dir() + and not Path(deploy_root, "symlinks/Grand1/Parent3").is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand1/Parent3/Child1").is_dir() + and not Path(deploy_root, "symlinks/Grand1/Parent3/Child1").is_symlink() + ) + assert ( + Path( + deploy_root, "symlinks/Grand1/Parent3/Child1/GrandChild2" + ).is_file() + and Path( + deploy_root, "symlinks/Grand1/Parent3/Child1/GrandChild2" + ).is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand2").is_dir() + and not Path(deploy_root, "symlinks/Grand2").is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand2/Parent1").is_dir() + and not Path(deploy_root, "symlinks/Grand2/Parent1").is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand2/Parent1/ChildA").is_dir() + and not Path(deploy_root, "symlinks/Grand2/Parent1/ChildA").is_symlink() + ) + assert ( + Path( + deploy_root, "symlinks/Grand2/Parent1/ChildA/GrandChildA" + ).is_file() + and Path( + deploy_root, "symlinks/Grand2/Parent1/ChildA/GrandChildA" + ).is_symlink() + ) + assert ( + Path(deploy_root, "symlinks/Grand2/Parent1/ChildB/GrandChildA").is_dir() + and not Path( + deploy_root, "symlinks/Grand2/Parent1/ChildB/GrandChildA" + ).is_symlink() + ) + + assert_dir_snapshot(Path("./output/deploy"), os_agnostic_snapshot) diff --git a/tests/nativeapp/__snapshots__/test_artifacts.ambr b/tests/nativeapp/__snapshots__/test_artifacts.ambr index 71ad43992f..ad2081f533 100644 --- a/tests/nativeapp/__snapshots__/test_artifacts.ambr +++ b/tests/nativeapp/__snapshots__/test_artifacts.ambr @@ -123,328 +123,3 @@ ''' # --- -# name: test_symlink_or_copy_raises_error - 'Test 1' -# --- -# name: test_symlink_or_copy_raises_error.1 - 'Test 1' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root - ''' - d . - d GrandA - d GrandA/ParentA - d GrandA/ParentA/ChildA - f GrandA/ParentA/ChildA/GrandChildA - f GrandA/ParentA/ChildA/GrandChildB.py - d GrandA/ParentA/ChildA/GrandChildC - f GrandA/ParentA/ChildB.py - f GrandA/ParentA/ChildC - d GrandA/ParentA/ChildD - d GrandA/ParentB - f GrandA/ParentB/ChildA - f GrandA/ParentB/ChildB.py - d GrandA/ParentB/ChildC - d GrandA/ParentB/ChildC/GrandChildA - d GrandA/ParentC - d GrandB - d GrandB/ParentA - f GrandB/ParentA/ChildA - d output - d output/deploy - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.1 - ''' - ===== Contents of: GrandA/ParentA/ChildA/GrandChildA ===== - Text GrandA/ParentA/ChildA/GrandChildA - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.10 - ''' - ===== Contents of: output/deploy/ChildA/GrandChildB.py ===== - Text GrandA/ParentA/ChildA/GrandChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.11 - ''' - ===== Contents of: output/deploy/ChildB.py ===== - Text GrandA/ParentA/ChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.12 - ''' - ===== Contents of: output/deploy/ChildC ===== - Text GrandA/ParentA/ChildC - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.13 - ''' - ===== Contents of: output/deploy/Grand1/Parent1/Child1 ===== - Text GrandB/ParentA/ChildA - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.14 - ''' - ===== Contents of: output/deploy/Grand3 ===== - Text GrandA/ParentB/ChildA - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.15 - ''' - ===== Contents of: output/deploy/Grand4/Parent1.py ===== - Text GrandA/ParentB/ChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.16 - ''' - ===== Contents of: output/deploy/Grand4/Parent3/ChildA/GrandChildA ===== - Text GrandA/ParentA/ChildA/GrandChildA - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.17 - ''' - ===== Contents of: output/deploy/Grand4/Parent3/ChildA/GrandChildB.py ===== - Text GrandA/ParentA/ChildA/GrandChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.18 - ''' - ===== Contents of: output/deploy/Grand4/Parent3/ChildB.py ===== - Text GrandA/ParentA/ChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.19 - ''' - ===== Contents of: output/deploy/Grand4/Parent3/ChildC ===== - Text GrandA/ParentA/ChildC - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.2 - ''' - ===== Contents of: GrandA/ParentA/ChildA/GrandChildB.py ===== - Text GrandA/ParentA/ChildA/GrandChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.3 - ''' - ===== Contents of: GrandA/ParentA/ChildB.py ===== - Text GrandA/ParentA/ChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.4 - ''' - ===== Contents of: GrandA/ParentA/ChildC ===== - Text GrandA/ParentA/ChildC - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.5 - ''' - ===== Contents of: GrandA/ParentB/ChildA ===== - Text GrandA/ParentB/ChildA - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.6 - ''' - ===== Contents of: GrandA/ParentB/ChildB.py ===== - Text GrandA/ParentB/ChildB.py - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.7 - ''' - ===== Contents of: GrandB/ParentA/ChildA ===== - Text GrandB/ParentA/ChildA - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.8 - ''' - d output/deploy - d output/deploy/ChildA - f output/deploy/ChildA/GrandChildA - f output/deploy/ChildA/GrandChildB.py - d output/deploy/ChildA/GrandChildC - f output/deploy/ChildB.py - f output/deploy/ChildC - d output/deploy/ChildD - d output/deploy/Grand1 - d output/deploy/Grand1/Parent1 - f output/deploy/Grand1/Parent1/Child1 - d output/deploy/Grand2 - f output/deploy/Grand3 - d output/deploy/Grand4 - f output/deploy/Grand4/Parent1.py - d output/deploy/Grand4/Parent2 - d output/deploy/Grand4/Parent2/GrandChildA - d output/deploy/Grand4/Parent3 - d output/deploy/Grand4/Parent3/ChildA - f output/deploy/Grand4/Parent3/ChildA/GrandChildA - f output/deploy/Grand4/Parent3/ChildA/GrandChildB.py - d output/deploy/Grand4/Parent3/ChildA/GrandChildC - f output/deploy/Grand4/Parent3/ChildB.py - f output/deploy/Grand4/Parent3/ChildC - d output/deploy/Grand4/Parent3/ChildD - ''' -# --- -# name: test_symlink_or_copy_with_no_symlinks_in_project_root.9 - ''' - ===== Contents of: output/deploy/ChildA/GrandChildA ===== - Text GrandA/ParentA/ChildA/GrandChildA - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root - ''' - d . - d GrandA - f GrandA/ParentA - f GrandA/ParentB - d GrandA/ParentC - d GrandA/ParentC/ChildA - f GrandA/ParentC/ChildA/GrandChildA - f GrandA/ParentC/ChildA/GrandChildB - d GrandB - d GrandB/ParentA - d GrandB/ParentA/ChildA - f GrandB/ParentA/ChildA/GrandChildA - d GrandB/ParentA/ChildB - d GrandB/ParentA/ChildB/GrandChildA - d output - d output/deploy - d symlinks - d symlinks/Grand1 - d symlinks/Grand1/Parent3 - d symlinks/Grand1/Parent3/Child1 - d symlinks/Grand2 - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.1 - ''' - ===== Contents of: GrandA/ParentA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.10 - ''' - ===== Contents of: output/deploy/TestA/ParentA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.11 - ''' - ===== Contents of: output/deploy/TestA/ParentB ===== - Use as src of a symlink: GrandA/ParentB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.12 - ''' - ===== Contents of: output/deploy/TestA/ParentC/ChildA/GrandChildA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.13 - ''' - ===== Contents of: output/deploy/TestA/ParentC/ChildA/GrandChildB ===== - Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.14 - ''' - ===== Contents of: output/deploy/TestB/ParentA/ChildA/GrandChildA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.15 - ''' - ===== Contents of: output/deploy/symlinks/Grand1/Parent2 ===== - Use as src of a symlink: GrandA/ParentB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.16 - ''' - ===== Contents of: output/deploy/symlinks/Grand1/Parent3/Child1/GrandChild2 ===== - Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.17 - ''' - ===== Contents of: output/deploy/symlinks/Grand2/Parent1/ChildA/GrandChildA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.2 - ''' - ===== Contents of: GrandA/ParentB ===== - Use as src of a symlink: GrandA/ParentB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.3 - ''' - ===== Contents of: GrandA/ParentC/ChildA/GrandChildA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.4 - ''' - ===== Contents of: GrandA/ParentC/ChildA/GrandChildB ===== - Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.5 - ''' - ===== Contents of: GrandB/ParentA/ChildA/GrandChildA ===== - Do not use as src of a symlink - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.6 - ''' - d symlinks - d symlinks/Grand1 - f symlinks/Grand1/Parent2 - d symlinks/Grand1/Parent3 - d symlinks/Grand1/Parent3/Child1 - f symlinks/Grand1/Parent3/Child1/GrandChild2 - d symlinks/Grand2 - d symlinks/Grand2/Parent1 - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.7 - ''' - ===== Contents of: symlinks/Grand1/Parent2 ===== - Use as src of a symlink: GrandA/ParentB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.8 - ''' - ===== Contents of: symlinks/Grand1/Parent3/Child1/GrandChild2 ===== - Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB - ''' -# --- -# name: test_symlink_or_copy_with_symlinks_in_project_root.9 - ''' - d output/deploy - d output/deploy/TestA - f output/deploy/TestA/ParentA - f output/deploy/TestA/ParentB - d output/deploy/TestA/ParentC - d output/deploy/TestA/ParentC/ChildA - f output/deploy/TestA/ParentC/ChildA/GrandChildA - f output/deploy/TestA/ParentC/ChildA/GrandChildB - d output/deploy/TestB - d output/deploy/TestB/ParentA - d output/deploy/TestB/ParentA/ChildA - f output/deploy/TestB/ParentA/ChildA/GrandChildA - d output/deploy/TestB/ParentA/ChildB - d output/deploy/TestB/ParentA/ChildB/GrandChildA - d output/deploy/symlinks - d output/deploy/symlinks/Grand1 - f output/deploy/symlinks/Grand1/Parent2 - d output/deploy/symlinks/Grand1/Parent3 - d output/deploy/symlinks/Grand1/Parent3/Child1 - f output/deploy/symlinks/Grand1/Parent3/Child1/GrandChild2 - d output/deploy/symlinks/Grand2 - d output/deploy/symlinks/Grand2/Parent1 - d output/deploy/symlinks/Grand2/Parent1/ChildA - f output/deploy/symlinks/Grand2/Parent1/ChildA/GrandChildA - d output/deploy/symlinks/Grand2/Parent1/ChildB - d output/deploy/symlinks/Grand2/Parent1/ChildB/GrandChildA - ''' -# --- diff --git a/tests/nativeapp/fixtures.py b/tests/nativeapp/fixtures.py index 390898d5f2..171c5bf154 100644 --- a/tests/nativeapp/fixtures.py +++ b/tests/nativeapp/fixtures.py @@ -16,7 +16,6 @@ import factory import pytest -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.nativeapp.entities.application import ( ApplicationEntity, ApplicationEntityModel, @@ -25,6 +24,7 @@ ApplicationPackageEntity, ApplicationPackageEntityModel, ) +from snowflake.cli.api.artifacts.bundle_map import BundleMap from tests.nativeapp.factories import ( ApplicationEntityModelFactory, diff --git a/tests/nativeapp/test_artifacts.py b/tests/nativeapp/test_artifacts.py index 59f141ad49..b0372d2e2b 100644 --- a/tests/nativeapp/test_artifacts.py +++ b/tests/nativeapp/test_artifacts.py @@ -14,27 +14,22 @@ from __future__ import annotations -import os -import re from pathlib import Path -from typing import Dict, Iterable, List, Optional, Union import pytest from click import ClickException from snowflake.cli._plugins.nativeapp.artifacts import ( + build_bundle, + find_events_definitions_in_manifest_file, + find_version_info_in_manifest_file, +) +from snowflake.cli.api.artifacts.common import ( ArtifactError, - ArtifactPredicate, - BundleMap, DeployRootError, NotInDeployRootError, SourceNotFoundError, TooManyFilesError, VersionInfo, - build_bundle, - find_events_definitions_in_manifest_file, - find_version_info_in_manifest_file, - resolve_without_follow, - symlink_or_copy, ) from snowflake.cli.api.project.definition import load_project from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping @@ -44,859 +39,8 @@ from tests.nativeapp.factories import ManifestFactory from tests.nativeapp.utils import ( assert_dir_snapshot, - touch, ) from tests.testing_utils.files_and_dirs import pushd, temp_local_dir -from tests_common import IS_WINDOWS - - -def trimmed_contents(path: Path) -> Optional[str]: - if not path.is_file(): - return None - with open(path, "r") as handle: - return handle.read().strip() - - -def dir_structure(path: Path, prefix="") -> List[str]: - if not path.is_dir(): - raise ValueError("Path must point to a directory") - - parts: List[str] = [] - for child in sorted(path.iterdir()): - if child.is_dir(): - parts += dir_structure(child, f"{prefix}{child.name}/") - else: - parts.append(f"{prefix}{child.name}") - - return parts - - -@pytest.fixture -def bundle_map(): - project_files = { - "snowflake.yml": "# empty", - "README.md": "# Test Project", - "app/setup.sql": "-- empty", - "app/manifest.yml": "# empty", - "src/snowpark/main.py": "# empty", - "src/snowpark/a/file1.py": "# empty", - "src/snowpark/a/file2.py": "# empty", - "src/snowpark/a/b/file3.py": "# empty", - "src/snowpark/a/b/file4.py": "# empty", - "src/snowpark/a/c/file5.py": "# empty", - "src/streamlit/main_ui.py": "# empty", - "src/streamlit/helpers/file1.py": "# empty", - "src/streamlit/helpers/file2.py": "# empty", - } - with temp_local_dir(project_files) as project_root: - deploy_root = project_root / "output" / "deploy" - yield BundleMap(project_root=project_root, deploy_root=deploy_root) - - -def ensure_path(path: Union[Path, str]) -> Path: - if isinstance(path, str): - return Path(path) - return path - - -def verify_mappings( - bundle_map: BundleMap, - expected_mappings: Dict[ - Union[str, Path], Optional[Union[str, Path, List[str], List[Path]]] - ], - expected_deploy_paths: ( - Dict[Union[str, Path], Optional[Union[str, Path, List[str], List[Path]]]] | None - ) = None, - **kwargs, -): - def normalize_expected_dest( - dest: Optional[Union[str, Path, List[str], List[Path]]] - ): - if dest is None: - return [] - elif isinstance(dest, str): - return [ensure_path(dest)] - elif isinstance(dest, Path): - return [dest] - else: - return sorted([ensure_path(d) for d in dest]) - - normalized_expected_mappings = { - ensure_path(src): normalize_expected_dest(dest) - for src, dest in expected_mappings.items() - if dest is not None - } - if expected_deploy_paths is not None: - normalized_expected_deploy_paths = { - ensure_path(src): normalize_expected_dest(dest) - for src, dest in expected_deploy_paths.items() - } - else: - normalized_expected_deploy_paths = normalized_expected_mappings - - for src, expected_dests in normalized_expected_deploy_paths.items(): - assert sorted(bundle_map.to_deploy_paths(ensure_path(src))) == expected_dests - - actual_path_mappings: Dict[Path, List[Path]] = {} - for src, dest in bundle_map.all_mappings(**kwargs): - mappings = actual_path_mappings.setdefault(src, []) - mappings.append(dest) - mappings.sort() - - assert actual_path_mappings == normalized_expected_mappings - - -def verify_sources( - bundle_map: BundleMap, expected_sources: Iterable[Union[str, Path]], **kwargs -) -> None: - actual_sources = sorted(bundle_map.all_sources(**kwargs)) - expected_sources = sorted([ensure_path(src) for src in expected_sources]) - assert actual_sources == expected_sources - - -def test_empty_bundle_map(bundle_map): - mappings = list(bundle_map.all_mappings()) - assert mappings == [] - - verify_sources(bundle_map, []) - - verify_mappings( - bundle_map, - { - "app/setup.sql": None, - ".": None, - "/not/in/project": None, - }, - ) - - -def test_bundle_map_requires_absolute_project_root(): - project_root = Path() - with pytest.raises( - AssertionError, - match=re.escape(rf"Project root {project_root} must be an absolute path."), - ): - BundleMap(project_root=project_root, deploy_root=Path("output/deploy")) - - -def test_bundle_map_requires_absolute_deploy_root(): - deploy_root = Path("output/deploy") - with pytest.raises( - AssertionError, - match=re.escape(rf"Deploy root {deploy_root} must be an absolute path."), - ): - BundleMap(project_root=Path().resolve(), deploy_root=deploy_root) - - -def test_bundle_map_handles_file_to_file_mappings(bundle_map): - bundle_map.add(PathMapping(src="README.md", dest="deployed_readme.md")) - bundle_map.add(PathMapping(src="app/setup.sql", dest="app_setup.sql")) - bundle_map.add(PathMapping(src="app/manifest.yml", dest="manifest.yml")) - - verify_mappings( - bundle_map, - { - "README.md": "deployed_readme.md", - "app/setup.sql": "app_setup.sql", - "app/manifest.yml": "manifest.yml", - }, - ) - - verify_sources(bundle_map, ["README.md", "app/setup.sql", "app/manifest.yml"]) - - -def test_bundle_map_supports_double_star_glob(bundle_map): - bundle_map.add(PathMapping(src="src/snowpark/**/*.py", dest="deployed/")) - - expected_mappings = { - "src/snowpark/main.py": "deployed/main.py", - "src/snowpark/a/file1.py": "deployed/file1.py", - "src/snowpark/a/file2.py": "deployed/file2.py", - "src/snowpark/a/b/file3.py": "deployed/file3.py", - "src/snowpark/a/b/file4.py": "deployed/file4.py", - "src/snowpark/a/c/file5.py": "deployed/file5.py", - } - - verify_mappings(bundle_map, expected_mappings) - - verify_sources(bundle_map, expected_mappings.keys()) - - -def test_bundle_map_supports_complex_globbing(bundle_map): - bundle_map.add(PathMapping(src="src/s*/**/file[3-5].py", dest="deployed/")) - - expected_mappings = { - "src/snowpark/main.py": None, - "src/snowpark/a/file1.py": None, - "src/snowpark/a/file2.py": None, - "src/snowpark/a/b/file3.py": "deployed/file3.py", - "src/snowpark/a/b/file4.py": "deployed/file4.py", - "src/snowpark/a/c/file5.py": "deployed/file5.py", - } - - verify_mappings( - bundle_map, - expected_mappings, - ) - - verify_sources( - bundle_map, - [src for src in expected_mappings.keys() if expected_mappings[src] is not None], - ) - - -def test_bundle_map_handles_mapping_to_deploy_root(bundle_map): - bundle_map.add(PathMapping(src="app/*", dest="./")) - bundle_map.add(PathMapping(src="README.md", dest="./")) - - verify_mappings( - bundle_map, - { - "app/setup.sql": "setup.sql", - "app/manifest.yml": "manifest.yml", - "README.md": "README.md", - }, - ) - - -def test_bundle_map_can_rename_directories(bundle_map): - bundle_map.add(PathMapping(src="app", dest="deployed")) - - verify_mappings( - bundle_map, - { - "app": "deployed", - }, - expand_directories=False, - ) - - verify_mappings( - bundle_map, - { - "app": "deployed", - "app/setup.sql": "deployed/setup.sql", - "app/manifest.yml": "deployed/manifest.yml", - }, - expand_directories=True, - ) - - -def test_bundle_map_honours_trailing_slashes(bundle_map): - bundle_map.add(PathMapping(src="app", dest="deployed/")) - bundle_map.add(PathMapping(src="README.md", dest="deployed/")) - bundle_map.add( - # src trailing slash has no effect - PathMapping(src="src/snowpark/", dest="deployed/") - ) - - verify_mappings( - bundle_map, - { - "app": "deployed/app", - "src/snowpark": "deployed/snowpark", - "README.md": "deployed/README.md", - }, - ) - - verify_mappings( - bundle_map, - { - "app": "deployed/app", - "app/manifest.yml": "deployed/app/manifest.yml", - "app/setup.sql": "deployed/app/setup.sql", - "src/snowpark": "deployed/snowpark", - "src/snowpark/main.py": "deployed/snowpark/main.py", - "src/snowpark/a": "deployed/snowpark/a", - "src/snowpark/a/file1.py": "deployed/snowpark/a/file1.py", - "src/snowpark/a/file2.py": "deployed/snowpark/a/file2.py", - "src/snowpark/a/b": "deployed/snowpark/a/b", - "src/snowpark/a/b/file3.py": "deployed/snowpark/a/b/file3.py", - "src/snowpark/a/b/file4.py": "deployed/snowpark/a/b/file4.py", - "src/snowpark/a/c": "deployed/snowpark/a/c", - "src/snowpark/a/c/file5.py": "deployed/snowpark/a/c/file5.py", - "README.md": "deployed/README.md", - }, - expand_directories=True, - ) - - -def test_bundle_map_disallows_overwriting_deploy_root(bundle_map): - with pytest.raises(NotInDeployRootError): - bundle_map.add(PathMapping(src="app/*", dest=".")) - - -def test_bundle_map_disallows_unknown_sources(bundle_map): - with pytest.raises(SourceNotFoundError): - bundle_map.add(PathMapping(src="missing/*", dest="deployed/")) - - with pytest.raises(SourceNotFoundError): - bundle_map.add(PathMapping(src="missing", dest="deployed/")) - - with pytest.raises(SourceNotFoundError): - bundle_map.add(PathMapping(src="**/*.missing", dest="deployed/")) - - -def test_bundle_map_disallows_mapping_multiple_to_file(bundle_map): - with pytest.raises(TooManyFilesError): - # multiple files named 'file1.py' would collide - bundle_map.add(PathMapping(src="**/file1.py", dest="deployed/")) - - with pytest.raises(TooManyFilesError): - bundle_map.add(PathMapping(src="**/file1.py", dest="deployed/")) - - -def test_bundle_map_allows_mapping_file_to_multiple_destinations(bundle_map): - bundle_map.add(PathMapping(src="README.md", dest="deployed/README1.md")) - bundle_map.add(PathMapping(src="README.md", dest="deployed/README2.md")) - bundle_map.add(PathMapping(src="src/streamlit", dest="deployed/streamlit_orig")) - bundle_map.add(PathMapping(src="src/streamlit", dest="deployed/streamlit_copy")) - bundle_map.add(PathMapping(src="src/streamlit/main_ui.py", dest="deployed/")) - - verify_mappings( - bundle_map, - expected_mappings={ - "README.md": ["deployed/README1.md", "deployed/README2.md"], - "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], - "src/streamlit/main_ui.py": ["deployed/main_ui.py"], - }, - expected_deploy_paths={ - "README.md": ["deployed/README1.md", "deployed/README2.md"], - "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], - "src/streamlit/main_ui.py": [ - "deployed/main_ui.py", - "deployed/streamlit_orig/main_ui.py", - "deployed/streamlit_copy/main_ui.py", - ], - }, - ) - - verify_mappings( - bundle_map, - expected_mappings={ - "README.md": ["deployed/README1.md", "deployed/README2.md"], - "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], - "src/streamlit/main_ui.py": [ - "deployed/main_ui.py", - "deployed/streamlit_orig/main_ui.py", - "deployed/streamlit_copy/main_ui.py", - ], - "src/streamlit/helpers": [ - "deployed/streamlit_orig/helpers", - "deployed/streamlit_copy/helpers", - ], - "src/streamlit/helpers/file1.py": [ - "deployed/streamlit_orig/helpers/file1.py", - "deployed/streamlit_copy/helpers/file1.py", - ], - "src/streamlit/helpers/file2.py": [ - "deployed/streamlit_orig/helpers/file2.py", - "deployed/streamlit_copy/helpers/file2.py", - ], - }, - expected_deploy_paths={ - "README.md": ["deployed/README1.md", "deployed/README2.md"], - "src/streamlit": ["deployed/streamlit_orig", "deployed/streamlit_copy"], - "src/streamlit/main_ui.py": [ - "deployed/main_ui.py", - "deployed/streamlit_orig/main_ui.py", - "deployed/streamlit_copy/main_ui.py", - ], - "src/streamlit/helpers": [ - "deployed/streamlit_orig/helpers", - "deployed/streamlit_copy/helpers", - ], - "src/streamlit/helpers/file1.py": [ - "deployed/streamlit_orig/helpers/file1.py", - "deployed/streamlit_copy/helpers/file1.py", - ], - "src/streamlit/helpers/file2.py": [ - "deployed/streamlit_orig/helpers/file2.py", - "deployed/streamlit_copy/helpers/file2.py", - ], - }, - expand_directories=True, - ) - - -def test_bundle_map_handles_missing_dest(bundle_map): - bundle_map.add(PathMapping(src="app")) - bundle_map.add(PathMapping(src="README.md")) - bundle_map.add(PathMapping(src="src/streamlit/")) - - verify_mappings( - bundle_map, - {"app": "app", "README.md": "README.md", "src/streamlit": "src/streamlit"}, - ) - - verify_mappings( - bundle_map, - { - "app": "app", - "app/setup.sql": "app/setup.sql", - "app/manifest.yml": "app/manifest.yml", - "README.md": "README.md", - "src/streamlit": "src/streamlit", - "src/streamlit/helpers": "src/streamlit/helpers", - "src/streamlit/main_ui.py": "src/streamlit/main_ui.py", - "src/streamlit/helpers/file1.py": "src/streamlit/helpers/file1.py", - "src/streamlit/helpers/file2.py": "src/streamlit/helpers/file2.py", - }, - expand_directories=True, - ) - - -def test_bundle_map_disallows_mapping_files_as_directories(bundle_map): - bundle_map.add(PathMapping(src="app", dest="deployed/")) - with pytest.raises( - ArtifactError, match="Conflicting type for destination path: deployed" - ): - bundle_map.add(PathMapping(src="**/main.py", dest="deployed")) - - -def test_bundle_map_disallows_mapping_directories_as_files(bundle_map): - bundle_map.add(PathMapping(src="**/main.py", dest="deployed")) - with pytest.raises( - ArtifactError, match="Conflicting type for destination path: deployed" - ): - bundle_map.add(PathMapping(src="app", dest="deployed")) - - -def test_bundle_map_allows_deploying_other_sources_to_renamed_directory(bundle_map): - bundle_map.add(PathMapping(src="src/snowpark", dest="./snowpark")) - bundle_map.add(PathMapping(src="README.md", dest="snowpark/")) - - verify_mappings( - bundle_map, - { - "src/snowpark": "snowpark", - "README.md": "snowpark/README.md", - }, - ) - - verify_mappings( - bundle_map, - { - "README.md": "snowpark/README.md", - "src/snowpark": "snowpark", - "src/snowpark/main.py": "snowpark/main.py", - "src/snowpark/a": "snowpark/a", - "src/snowpark/a/file1.py": "snowpark/a/file1.py", - "src/snowpark/a/file2.py": "snowpark/a/file2.py", - "src/snowpark/a/b": "snowpark/a/b", - "src/snowpark/a/b/file3.py": "snowpark/a/b/file3.py", - "src/snowpark/a/b/file4.py": "snowpark/a/b/file4.py", - "src/snowpark/a/c": "snowpark/a/c", - "src/snowpark/a/c/file5.py": "snowpark/a/c/file5.py", - }, - expand_directories=True, - ) - - -@pytest.mark.skip(reason="Checking deep tree hierarchies is not yet supported") -def test_bundle_map_disallows_collisions_anywhere_in_deployed_hierarchy(bundle_map): - bundle_map.add(PathMapping(src="src/snowpark", dest="./snowpark")) - bundle_map.add(PathMapping(src="README.md", dest="snowpark/")) - - # if any of the files collide, however, this is not allowed - with pytest.raises(TooManyFilesError): - bundle_map.add(PathMapping(src="app/manifest.yml", dest="snowpark/README.md")) - - with pytest.raises(TooManyFilesError): - bundle_map.add(PathMapping(src="app/manifest.yml", dest="snowpark/a/file1.py")) - - -def test_bundle_map_disallows_mapping_outside_deploy_root(bundle_map): - with pytest.raises(NotInDeployRootError): - bundle_map.add(PathMapping(src="app", dest="deployed/../../")) - - with pytest.raises(NotInDeployRootError): - bundle_map.add(PathMapping(src="app", dest=Path().resolve().root)) - - with pytest.raises(NotInDeployRootError): - bundle_map.add(PathMapping(src="app", dest="/////")) - - -def test_bundle_map_disallows_absolute_src(bundle_map): - with pytest.raises(ArtifactError): - absolute_src = bundle_map.project_root() / "app" - assert absolute_src.is_absolute() - bundle_map.add(PathMapping(src=str(absolute_src), dest="deployed")) - - -def test_bundle_map_disallows_absolute_dest(bundle_map): - with pytest.raises(ArtifactError): - absolute_dest = bundle_map.deploy_root() / "deployed" - assert absolute_dest.is_absolute() - bundle_map.add(PathMapping(src="app", dest=str(absolute_dest))) - - -def test_bundle_map_disallows_clobbering_parent_directories(bundle_map): - # one level of nesting - with pytest.raises(TooManyFilesError): - bundle_map.add(PathMapping(src="snowflake.yml", dest="./app/")) - # Adding a new rule to populate ./app/ from an existing directory. This would - # clobber the output of the previous rule, so it's disallowed - bundle_map.add(PathMapping(src="./app", dest="./")) - - # same as above but with multiple levels of nesting - with pytest.raises(TooManyFilesError): - bundle_map.add(PathMapping(src="snowflake.yml", dest="./src/snowpark/a/")) - bundle_map.add(PathMapping(src="./src/snowpark", dest="./src/")) - - -def test_bundle_map_disallows_clobbering_child_directories(bundle_map): - with pytest.raises(TooManyFilesError): - bundle_map.add(PathMapping(src="./src/snowpark", dest="./python/")) - bundle_map.add(PathMapping(src="./app", dest="./python/snowpark/a")) - - -def test_bundle_map_allows_augmenting_dest_directories(bundle_map): - # one level of nesting - # First populate {deploy}/app from an existing directory - bundle_map.add(PathMapping(src="./app", dest="./")) - # Then add a new file to that directory - bundle_map.add(PathMapping(src="snowflake.yml", dest="./app/")) - - # verify that when iterating over mappings, the base directory rule appears first, - # followed by the file. This is important for correctness, and should be - # deterministic - ordered_dests = [ - dest for (_, dest) in bundle_map.all_mappings(expand_directories=True) - ] - file_index = ordered_dests.index(Path("app/snowflake.yml")) - dir_index = ordered_dests.index(Path("app")) - assert dir_index < file_index - - -def test_bundle_map_allows_augmenting_dest_directories_nested(bundle_map): - # same as above but with multiple levels of nesting - bundle_map.add(PathMapping(src="./src/snowpark", dest="./src/")) - bundle_map.add(PathMapping(src="snowflake.yml", dest="./src/snowpark/a/")) - - ordered_dests = [ - dest for (_, dest) in bundle_map.all_mappings(expand_directories=True) - ] - file_index = ordered_dests.index(Path("src/snowpark/a/snowflake.yml")) - dir_index = ordered_dests.index(Path("src/snowpark")) - assert dir_index < file_index - - -def test_bundle_map_returns_mappings_in_insertion_order(bundle_map): - # this behaviour is important to make sure the deploy root is populated in a - # deterministic manner, so verify it here - bundle_map.add(PathMapping(src="./app", dest="./")) - bundle_map.add(PathMapping(src="snowflake.yml", dest="./app/")) - bundle_map.add(PathMapping(src="./src/snowpark", dest="./src/")) - bundle_map.add(PathMapping(src="snowflake.yml", dest="./src/snowpark/a/")) - - ordered_dests = [ - dest for (_, dest) in bundle_map.all_mappings(expand_directories=False) - ] - assert ordered_dests == [ - Path("app"), - Path("app/snowflake.yml"), - Path("src/snowpark"), - Path("src/snowpark/a/snowflake.yml"), - ] - - -def test_bundle_map_all_mappings_generates_absolute_directories_when_requested( - bundle_map, -): - project_root = bundle_map.project_root() - assert project_root.is_absolute() - deploy_root = bundle_map.deploy_root() - assert deploy_root.is_absolute() - - bundle_map.add(PathMapping(src="app", dest="deployed_app")) - bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) - bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) - - verify_mappings( - bundle_map, - { - "app": "deployed_app", - "README.md": "deployed_README.md", - "src/streamlit": "deployed_streamlit", - }, - ) - - verify_mappings( - bundle_map, - { - project_root / "app": deploy_root / "deployed_app", - project_root / "README.md": deploy_root / "deployed_README.md", - project_root / "src/streamlit": deploy_root / "deployed_streamlit", - }, - absolute=True, - expand_directories=False, - ) - - verify_mappings( - bundle_map, - { - project_root / "app": deploy_root / "deployed_app", - project_root / "app/setup.sql": deploy_root / "deployed_app/setup.sql", - project_root - / "app/manifest.yml": deploy_root - / "deployed_app/manifest.yml", - project_root / "README.md": deploy_root / "deployed_README.md", - project_root / "src/streamlit": deploy_root / "deployed_streamlit", - project_root - / "src/streamlit/helpers": deploy_root - / "deployed_streamlit/helpers", - project_root - / "src/streamlit/main_ui.py": deploy_root - / "deployed_streamlit/main_ui.py", - project_root - / "src/streamlit/helpers/file1.py": deploy_root - / "deployed_streamlit/helpers/file1.py", - project_root - / "src/streamlit/helpers/file2.py": deploy_root - / "deployed_streamlit/helpers/file2.py", - }, - absolute=True, - expand_directories=True, - ) - - -def test_bundle_map_all_sources_generates_absolute_directories_when_requested( - bundle_map, -): - project_root = bundle_map.project_root() - assert project_root.is_absolute() - - bundle_map.add(PathMapping(src="app", dest="deployed_app")) - bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) - bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) - - verify_sources(bundle_map, ["app", "README.md", "src/streamlit"]) - - verify_sources( - bundle_map, - [ - project_root / "app", - project_root / "README.md", - project_root / "src/streamlit", - ], - absolute=True, - ) - - -def test_bundle_map_all_mappings_accepts_predicates(bundle_map): - project_root = bundle_map.project_root() - assert project_root.is_absolute() - deploy_root = bundle_map.deploy_root() - assert deploy_root.is_absolute() - - bundle_map.add(PathMapping(src="app", dest="deployed_app")) - bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) - bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) - - collected: Dict[Path, Path] = {} - - def collecting_predicate(predicate: ArtifactPredicate) -> ArtifactPredicate: - def _predicate(src: Path, dest: Path) -> bool: - collected[src] = dest - return predicate(src, dest) - - return _predicate - - verify_mappings( - bundle_map, - { - project_root - / "src/streamlit/main_ui.py": deploy_root - / "deployed_streamlit/main_ui.py", - project_root - / "src/streamlit/helpers/file1.py": deploy_root - / "deployed_streamlit/helpers/file1.py", - project_root - / "src/streamlit/helpers/file2.py": deploy_root - / "deployed_streamlit/helpers/file2.py", - }, - absolute=True, - expand_directories=True, - predicate=collecting_predicate( - lambda src, dest: src.is_file() and src.suffix == ".py" - ), - ) - - assert collected == { - project_root / "app": deploy_root / "deployed_app", - project_root / "app/setup.sql": deploy_root / "deployed_app/setup.sql", - project_root / "app/manifest.yml": deploy_root / "deployed_app/manifest.yml", - project_root / "README.md": deploy_root / "deployed_README.md", - project_root / "src/streamlit": deploy_root / "deployed_streamlit", - project_root - / "src/streamlit/helpers": deploy_root - / "deployed_streamlit/helpers", - project_root - / "src/streamlit/main_ui.py": deploy_root - / "deployed_streamlit/main_ui.py", - project_root - / "src/streamlit/helpers/file1.py": deploy_root - / "deployed_streamlit/helpers/file1.py", - project_root - / "src/streamlit/helpers/file2.py": deploy_root - / "deployed_streamlit/helpers/file2.py", - } - - collected = {} - - verify_mappings( - bundle_map, - { - "src/streamlit/main_ui.py": "deployed_streamlit/main_ui.py", - "src/streamlit/helpers/file1.py": "deployed_streamlit/helpers/file1.py", - "src/streamlit/helpers/file2.py": "deployed_streamlit/helpers/file2.py", - }, - absolute=False, - expand_directories=True, - predicate=collecting_predicate(lambda src, dest: src.suffix == ".py"), - ) - - assert collected == { - Path("app"): Path("deployed_app"), - Path("app/setup.sql"): Path("deployed_app/setup.sql"), - Path("app/manifest.yml"): Path("deployed_app/manifest.yml"), - Path("README.md"): Path("deployed_README.md"), - Path("src/streamlit"): Path("deployed_streamlit"), - Path("src/streamlit/main_ui.py"): Path("deployed_streamlit/main_ui.py"), - Path("src/streamlit/helpers"): Path("deployed_streamlit/helpers"), - Path("src/streamlit/helpers/file1.py"): Path( - "deployed_streamlit/helpers/file1.py" - ), - Path("src/streamlit/helpers/file2.py"): Path( - "deployed_streamlit/helpers/file2.py" - ), - } - - -def test_bundle_map_to_deploy_path(bundle_map): - bundle_map.add(PathMapping(src="app", dest="deployed_app")) - bundle_map.add(PathMapping(src="README.md", dest="deployed_README.md")) - bundle_map.add(PathMapping(src="src/streamlit", dest="deployed_streamlit")) - - # to_deploy_path returns relative paths when relative paths are given as input - assert bundle_map.to_deploy_paths(Path("app")) == [Path("deployed_app")] - assert bundle_map.to_deploy_paths(Path("README.md")) == [Path("deployed_README.md")] - assert bundle_map.to_deploy_paths(Path("src/streamlit")) == [ - Path("deployed_streamlit") - ] - assert bundle_map.to_deploy_paths(Path("src/streamlit/main_ui.py")) == [ - Path("deployed_streamlit/main_ui.py") - ] - assert bundle_map.to_deploy_paths(Path("src/streamlit/helpers")) == [ - Path("deployed_streamlit/helpers") - ] - assert bundle_map.to_deploy_paths(Path("src/streamlit/helpers/file1.py")) == [ - Path("deployed_streamlit/helpers/file1.py") - ] - assert bundle_map.to_deploy_paths(Path("src/streamlit/missing.py")) == [] - assert bundle_map.to_deploy_paths(Path("missing")) == [] - assert bundle_map.to_deploy_paths(Path("src/missing/")) == [] - assert bundle_map.to_deploy_paths(bundle_map.project_root().parent) == [] - - # to_deploy_path returns absolute paths when absolute paths are given as input - project_root = bundle_map.project_root() - deploy_root = bundle_map.deploy_root() - assert bundle_map.to_deploy_paths(project_root / "app") == [ - deploy_root / "deployed_app" - ] - assert bundle_map.to_deploy_paths(project_root / "README.md") == [ - deploy_root / "deployed_README.md" - ] - assert bundle_map.to_deploy_paths(project_root / "src/streamlit") == [ - deploy_root / "deployed_streamlit" - ] - assert bundle_map.to_deploy_paths(project_root / "src/streamlit/main_ui.py") == [ - deploy_root / "deployed_streamlit/main_ui.py" - ] - assert bundle_map.to_deploy_paths(project_root / "src/streamlit/helpers") == [ - deploy_root / "deployed_streamlit/helpers" - ] - assert bundle_map.to_deploy_paths( - project_root / "src/streamlit/helpers/file1.py" - ) == [deploy_root / "deployed_streamlit/helpers/file1.py"] - assert bundle_map.to_deploy_paths(project_root / "src/streamlit/missing.py") == [] - - -def test_bundle_map_to_deploy_path_returns_multiple_matches(bundle_map): - bundle_map.add(PathMapping(src="src/snowpark", dest="d1")) - bundle_map.add(PathMapping(src="src/snowpark", dest="d2")) - - assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark"))) == [ - Path("d1"), - Path("d2"), - ] - - assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark/main.py"))) == [ - Path("d1/main.py"), - Path("d2/main.py"), - ] - - assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark/a/b"))) == [ - Path("d1/a/b"), - Path("d2/a/b"), - ] - - bundle_map.add(PathMapping(src="src/snowpark/a", dest="d3")) - - assert sorted(bundle_map.to_deploy_paths(Path("src/snowpark/a/b/file3.py"))) == [ - Path("d1/a/b/file3.py"), - Path("d2/a/b/file3.py"), - Path("d3/b/file3.py"), - ] - - -@pytest.mark.parametrize( - "dest, src", - [ - ["manifest.yml", "app/manifest.yml"], - [".", None], - ["python/snowpark/main.py", "src/snowpark/main.py"], - ["python/snowpark", "src/snowpark"], - ["python/snowpark/a/b", "src/snowpark/a/b"], - ["python/snowpark/a/b/fake.py", None], - [ - # even though a rule creates this directory, it has no equivalent source folder - "python", - None, - ], - ["/fake/foo.py", None], - ], -) -def test_to_project_path(bundle_map, dest, src): - bundle_map.add(PathMapping(src="app/*", dest="./")) - bundle_map.add(PathMapping(src="src/snowpark", dest="./python/snowpark")) - - # relative paths - if src is None: - assert bundle_map.to_project_path(Path(dest)) is None - assert bundle_map.to_project_path(Path(bundle_map.deploy_root() / dest)) is None - else: - assert bundle_map.to_project_path(Path(dest)) == Path(src) - assert ( - bundle_map.to_project_path(Path(bundle_map.deploy_root() / dest)) - == bundle_map.project_root() / src - ) - - -def test_bundle_map_ignores_sources_in_deploy_root(bundle_map): - bundle_map.deploy_root().mkdir(parents=True, exist_ok=True) - deploy_root_source = bundle_map.deploy_root() / "should_not_match.yml" - touch(str(deploy_root_source)) - - bundle_map.add(PathMapping(src="**/*.yml", dest="deployed/")) - - verify_mappings( - bundle_map, - { - "app/manifest.yml": "deployed/manifest.yml", - "snowflake.yml": "deployed/snowflake.yml", - }, - ) @pytest.mark.parametrize("project_definition_files", ["napp_project_1"], indirect=True) @@ -1016,381 +160,6 @@ def test_too_many_files(project_definition_files): ) -@pytest.mark.skipif( - IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" -) -@pytest.mark.parametrize( - "project_path,expected_path", - [ - [ - "srcfile", - "deploy/file", - ], - [ - "srcdir", - "deploy/dir", - ], - [ - "srcdir/nested_file1", - "deploy/dir/nested_file1", - ], - [ - "srcdir/nested_dir/nested_file2", - "deploy/dir/nested_dir/nested_file2", - ], - [ - "srcdir/nested_dir", - "deploy/dir/nested_dir", - ], - [ - "not-in-deploy", - None, - ], - ], -) -def test_source_path_to_deploy_path( - temp_dir, - project_path, - expected_path, -): - # Source files - touch("srcfile") - touch("srcdir/nested_file1") - touch("srcdir/nested_dir/nested_file2") - touch("not-in-deploy") - # Build - os.mkdir("deploy") - os.symlink("srcfile", "deploy/file") - os.symlink(Path("srcdir").resolve(), Path("deploy/dir")) - - bundle_map = BundleMap( - project_root=Path().resolve(), deploy_root=Path("deploy").resolve() - ) - bundle_map.add(PathMapping(src="srcdir", dest="./dir")) - bundle_map.add(PathMapping(src="srcfile", dest="./file")) - - result = bundle_map.to_deploy_paths(resolve_without_follow(Path(project_path))) - if expected_path: - assert result == [resolve_without_follow(Path(expected_path))] - else: - assert result == [] - - -@pytest.mark.skipif( - IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" -) -def test_symlink_or_copy_raises_error(temp_dir, os_agnostic_snapshot): - touch("GrandA/ParentA/ChildA") - with open(Path(temp_dir, "GrandA/ParentA/ChildA"), "w") as f: - f.write("Test 1") - - # Create the deploy root - deploy_root = Path(temp_dir, "output", "deploy") - os.makedirs(deploy_root) - - # Incorrect dst path - with pytest.raises(NotInDeployRootError): - symlink_or_copy( - src=Path("GrandA", "ParentA", "ChildA"), - dst=Path("output", "ParentA", "ChildA"), - deploy_root=deploy_root, - ) - - file_in_deploy_root = Path("output", "deploy", "ParentA", "ChildA") - - # Correct path and parent directories are automatically created - symlink_or_copy( - src=Path("GrandA", "ParentA", "ChildA"), - dst=file_in_deploy_root, - deploy_root=deploy_root, - ) - - assert file_in_deploy_root.exists() and file_in_deploy_root.is_symlink() - assert file_in_deploy_root.read_text(encoding="utf-8") == os_agnostic_snapshot - - # Since file_in_deploy_root is a symlink - # it resolves to project_dir/GrandA/ParentA/ChildA, which is not in deploy root - with pytest.raises(NotInDeployRootError): - symlink_or_copy( - src=Path("GrandA", "ParentA", "ChildA"), - dst=file_in_deploy_root, - deploy_root=deploy_root, - ) - - # Unlink the symlink file and create a file with the same name and path - # This should pass since src.is_file() always begins by deleting the dst. - os.unlink(file_in_deploy_root) - touch(file_in_deploy_root) - symlink_or_copy( - src=Path("GrandA", "ParentA", "ChildA"), - dst=file_in_deploy_root, - deploy_root=deploy_root, - ) - - # dst is an existing symlink, will resolve to the src during NotInDeployRootError check. - touch("GrandA/ParentA/ChildB") - with pytest.raises(NotInDeployRootError): - symlink_or_copy( - src=Path("GrandA/ParentA/ChildB"), - dst=file_in_deploy_root, - deploy_root=deploy_root, - ) - assert file_in_deploy_root.exists() and file_in_deploy_root.is_symlink() - assert file_in_deploy_root.read_text(encoding="utf-8") == os_agnostic_snapshot - - -@pytest.mark.skipif( - IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" -) -def test_symlink_or_copy_with_no_symlinks_in_project_root(os_agnostic_snapshot): - test_dir_structure = { - "GrandA/ParentA/ChildA/GrandChildA": "Text GrandA/ParentA/ChildA/GrandChildA", - "GrandA/ParentA/ChildA/GrandChildB.py": "Text GrandA/ParentA/ChildA/GrandChildB.py", - "GrandA/ParentA/ChildA/GrandChildC": None, # dir - "GrandA/ParentA/ChildB.py": "Text GrandA/ParentA/ChildB.py", - "GrandA/ParentA/ChildC": "Text GrandA/ParentA/ChildC", - "GrandA/ParentA/ChildD": None, # dir - "GrandA/ParentB/ChildA": "Text GrandA/ParentB/ChildA", - "GrandA/ParentB/ChildB.py": "Text GrandA/ParentB/ChildB.py", - "GrandA/ParentB/ChildC/GrandChildA": None, # dir - "GrandA/ParentC": None, # dir - "GrandB/ParentA/ChildA": "Text GrandB/ParentA/ChildA", - "output/deploy": None, # dir - } - with temp_local_dir(test_dir_structure) as project_root: - with pushd(project_root): - # Sanity Check - assert_dir_snapshot(Path("."), os_agnostic_snapshot) - - deploy_root = Path(project_root, "output/deploy") - - # "GrandB" dir - symlink_or_copy( - src=Path("GrandB/ParentA/ChildA"), - dst=Path(deploy_root, "Grand1/Parent1/Child1"), - deploy_root=deploy_root, - ) - assert not Path(deploy_root, "Grand1").is_symlink() - assert not Path(deploy_root, "Grand1/Parent1").is_symlink() - assert Path(deploy_root, "Grand1/Parent1/Child1").is_symlink() - - # "GrandA/ParentC" dir - symlink_or_copy( - src=Path("GrandA/ParentC"), - dst=Path(deploy_root, "Grand2"), - deploy_root=deploy_root, - ) - assert not Path(deploy_root, "Grand2").is_symlink() - - # "GrandA/ParentB" dir - symlink_or_copy( - src=Path("GrandA/ParentB/ChildA"), - dst=Path(deploy_root, "Grand3"), - deploy_root=deploy_root, - ) - assert Path(deploy_root, "Grand3").is_symlink() - symlink_or_copy( - src=Path("GrandA/ParentB/ChildB.py"), - dst=Path(deploy_root, "Grand4/Parent1.py"), - deploy_root=deploy_root, - ) - assert not Path(deploy_root, "Grand4").is_symlink() - assert Path(deploy_root, "Grand4/Parent1.py").is_symlink() - symlink_or_copy( - src=Path("GrandA/ParentB/ChildC"), - dst=Path(deploy_root, "Grand4/Parent2"), - deploy_root=deploy_root, - ) - assert not Path(deploy_root, "Grand4").is_symlink() - assert not Path(deploy_root, "Grand4/Parent2").is_symlink() - assert not Path(deploy_root, "Grand4/Parent2/GrandChildA").is_symlink() - - # "GrandA/ParentA" dir (1) - symlink_or_copy( - src=Path("GrandA/ParentA"), dst=deploy_root, deploy_root=deploy_root - ) - assert not deploy_root.is_symlink() - assert not Path(deploy_root, "ChildA").is_symlink() - assert Path(deploy_root, "ChildA/GrandChildA").is_symlink() - assert Path(deploy_root, "ChildA/GrandChildB.py").is_symlink() - assert not Path(deploy_root, "ChildA/GrandChildC").is_symlink() - assert Path(deploy_root, "ChildB.py").is_symlink() - assert Path(deploy_root, "ChildC").is_symlink() - assert not Path(deploy_root, "ChildD").is_symlink() - - # "GrandA/ParentA" dir (2) - symlink_or_copy( - src=Path("GrandA/ParentA"), - dst=Path(deploy_root, "Grand4/Parent3"), - deploy_root=deploy_root, - ) - # Other children of Grand4 will be verified by a full assert_dir_snapshot(project_root) below - assert not Path(deploy_root, "Grand4/Parent3").is_symlink() - assert not Path(deploy_root, "Grand4/Parent3/ChildA").is_symlink() - assert Path(deploy_root, "Grand4/Parent3/ChildA/GrandChildA").is_symlink() - assert Path( - deploy_root, "Grand4/Parent3/ChildA/GrandChildB.py" - ).is_symlink() - assert not Path( - deploy_root, "Grand4/Parent3/ChildA/GrandChildC" - ).is_symlink() - assert Path(deploy_root, "Grand4/Parent3/ChildB.py").is_symlink() - assert Path(deploy_root, "Grand4/Parent3/ChildC").is_symlink() - assert not Path(deploy_root, "Grand4/Parent3/ChildD").is_symlink() - - assert_dir_snapshot(Path("./output/deploy"), os_agnostic_snapshot) - - # This is because the dst can be symlinks, which resolves to project src and hence outside deploy root. - with pytest.raises(NotInDeployRootError): - symlink_or_copy( - src=Path("GrandA/ParentB"), - dst=Path(deploy_root, "Grand4/Parent3"), - deploy_root=deploy_root, - ) - - -@pytest.mark.skipif( - IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" -) -def test_symlink_or_copy_with_symlinks_in_project_root(os_agnostic_snapshot): - test_dir_structure = { - "GrandA/ParentA": "Do not use as src of a symlink", - "GrandA/ParentB": "Use as src of a symlink: GrandA/ParentB", - "GrandA/ParentC/ChildA/GrandChildA": "Do not use as src of a symlink", - "GrandA/ParentC/ChildA/GrandChildB": "Use as src of a symlink: GrandA/ParentC/ChildA/GrandChildB", - "GrandB/ParentA/ChildA/GrandChildA": "Do not use as src of a symlink", - "GrandB/ParentA/ChildB/GrandChildA": None, - "symlinks/Grand1/Parent3/Child1": None, - "symlinks/Grand2": None, - "output/deploy": None, # dir - } - with temp_local_dir(test_dir_structure) as project_root: - with pushd(project_root): - # Sanity Check - assert_dir_snapshot(Path("."), os_agnostic_snapshot) - - os.symlink( - Path("GrandA/ParentB").resolve(), - Path(project_root, "symlinks/Grand1/Parent2"), - ) - os.symlink( - Path("GrandA/ParentC/ChildA/GrandChildB").resolve(), - Path(project_root, "symlinks/Grand1/Parent3/Child1/GrandChild2"), - ) - os.symlink( - Path("GrandB/ParentA").resolve(), - Path(project_root, "symlinks/Grand2/Parent1"), - target_is_directory=True, - ) - assert Path("symlinks").is_dir() and not Path("symlinks").is_symlink() - assert ( - Path("GrandA/ParentB").is_file() - and not Path("GrandA/ParentB").is_symlink() - ) - assert ( - Path("symlinks/Grand1/Parent2").is_symlink() - and Path("symlinks/Grand1/Parent2").is_file() - ) - assert ( - Path("symlinks/Grand1/Parent3/Child1/GrandChild2").is_symlink() - and Path("symlinks/Grand1/Parent3/Child1/GrandChild2").is_file() - ) - assert ( - Path("symlinks/Grand2/Parent1").is_symlink() - and Path("symlinks/Grand2/Parent1").is_dir() - ) - - # Sanity Check - assert_dir_snapshot(Path("./symlinks"), os_agnostic_snapshot) - - deploy_root = Path(project_root, "output/deploy") - - symlink_or_copy( - src=Path("GrandA"), - dst=Path(deploy_root, "TestA"), - deploy_root=deploy_root, - ) - assert not Path(deploy_root, "TestA").is_symlink() - assert Path(deploy_root, "TestA/ParentA").is_symlink() - assert Path(deploy_root, "TestA/ParentB").is_symlink() - assert not Path(deploy_root, "TestA/ParentC").is_symlink() - assert not Path(deploy_root, "TestA/ParentC/ChildA").is_symlink() - assert Path(deploy_root, "TestA/ParentC/ChildA/GrandChildA").is_symlink() - assert Path(deploy_root, "TestA/ParentC/ChildA/GrandChildB").is_symlink() - - symlink_or_copy( - src=Path("GrandB"), - dst=Path(deploy_root, "TestB"), - deploy_root=deploy_root, - ) - assert not Path(deploy_root, "TestB").is_symlink() - assert not Path(deploy_root, "TestB/ParentA").is_symlink() - assert not Path(deploy_root, "TestB/ParentA/ChildA").is_symlink() - assert not Path(deploy_root, "TestB/ParentA/ChildB").is_symlink() - assert not Path( - deploy_root, "TestB/ParentA/ChildB/GrandChildA" - ).is_symlink() - assert Path(deploy_root, "TestB/ParentA/ChildA/GrandChildA").is_symlink() - - symlink_or_copy( - src=Path("symlinks"), - dst=Path(deploy_root, "symlinks"), - deploy_root=deploy_root, - ) - assert ( - Path(deploy_root, "symlinks/Grand1").is_dir() - and not Path(deploy_root, "symlinks/Grand1").is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand1/Parent2").is_file() - and Path(deploy_root, "symlinks/Grand1/Parent2").is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand1/Parent3").is_dir() - and not Path(deploy_root, "symlinks/Grand1/Parent3").is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand1/Parent3/Child1").is_dir() - and not Path(deploy_root, "symlinks/Grand1/Parent3/Child1").is_symlink() - ) - assert ( - Path( - deploy_root, "symlinks/Grand1/Parent3/Child1/GrandChild2" - ).is_file() - and Path( - deploy_root, "symlinks/Grand1/Parent3/Child1/GrandChild2" - ).is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand2").is_dir() - and not Path(deploy_root, "symlinks/Grand2").is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand2/Parent1").is_dir() - and not Path(deploy_root, "symlinks/Grand2/Parent1").is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand2/Parent1/ChildA").is_dir() - and not Path(deploy_root, "symlinks/Grand2/Parent1/ChildA").is_symlink() - ) - assert ( - Path( - deploy_root, "symlinks/Grand2/Parent1/ChildA/GrandChildA" - ).is_file() - and Path( - deploy_root, "symlinks/Grand2/Parent1/ChildA/GrandChildA" - ).is_symlink() - ) - assert ( - Path(deploy_root, "symlinks/Grand2/Parent1/ChildB/GrandChildA").is_dir() - and not Path( - deploy_root, "symlinks/Grand2/Parent1/ChildB/GrandChildA" - ).is_symlink() - ) - - assert_dir_snapshot(Path("./output/deploy"), os_agnostic_snapshot) - - @pytest.mark.parametrize( "label", [None, "", "label 'nested' quotes", "with$pecial?", "with space"] ) diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index c61467c044..c729faec76 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -24,7 +24,6 @@ import pytest from click import ClickException -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.nativeapp.constants import ( LOOSE_FILES_MAGIC_VERSION, NAME_COL, @@ -52,6 +51,7 @@ StagePathType, ) from snowflake.cli._plugins.workspace.manager import WorkspaceManager +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.entities.common import EntityActions from snowflake.cli.api.entities.utils import ( diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index 8ee1e462fd..4cf3a9f3ed 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -21,7 +21,6 @@ from unittest import mock import pytest -from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.stage.diff import ( DiffResult, StagePathType, @@ -36,6 +35,7 @@ ) from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli._plugins.stage.utils import print_diff_to_console +from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.exceptions import ( SnowflakeSQLExecutionError, ) From caf9f1b48f5450759d2a44304eb1e331e04f5310 Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Wed, 27 Nov 2024 10:09:40 -0500 Subject: [PATCH 04/10] Move PathMapping and ProcessorMapping out of native app logic (#1899) --- .../cli/_plugins/nativeapp/artifacts.py | 2 +- .../cli/_plugins/nativeapp/bundle_context.py | 2 +- .../nativeapp/codegen/artifact_processor.py | 2 +- .../_plugins/nativeapp/codegen/compiler.py | 4 +- .../setup/native_app_setup_processor.py | 2 +- .../codegen/snowpark/python_processor.py | 2 +- .../codegen/templates/templates_processor.py | 2 +- .../nativeapp/entities/application_package.py | 28 +------ .../cli/_plugins/snowpark/commands.py | 2 +- .../snowpark/snowpark_entity_model.py | 4 +- .../snowpark/snowpark_project_paths.py | 2 +- .../cli/_plugins/streamlit/manager.py | 2 +- .../streamlit/streamlit_entity_model.py | 4 +- src/snowflake/cli/api/artifacts/bundle_map.py | 2 +- .../cli/api/project/schemas/commons.py | 5 -- .../api/project/schemas/entities/common.py | 75 ++++++++++++++++++- .../schemas/v1/native_app/native_app.py | 3 +- .../schemas/v1/native_app/path_mapping.py | 65 ---------------- tests/api/artifacts/test_bundle_map.py | 2 +- .../helpers/__snapshots__/test_v1_to_v2.ambr | 8 +- .../codegen/snowpark/test_python_processor.py | 4 +- .../templating/test_templates_processor.py | 2 +- .../test_annotation_processor_config.py | 4 +- tests/nativeapp/test_artifacts.py | 2 +- tests/project/test_config.py | 2 +- tests/project/test_project_definition_v2.py | 2 +- 26 files changed, 103 insertions(+), 131 deletions(-) delete mode 100644 src/snowflake/cli/api/project/schemas/commons.py delete mode 100644 src/snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py diff --git a/src/snowflake/cli/_plugins/nativeapp/artifacts.py b/src/snowflake/cli/_plugins/nativeapp/artifacts.py index 4e38faf9d6..bca39b5af2 100644 --- a/src/snowflake/cli/_plugins/nativeapp/artifacts.py +++ b/src/snowflake/cli/_plugins/nativeapp/artifacts.py @@ -25,7 +25,7 @@ from snowflake.cli.api.artifacts.utils import symlink_or_copy from snowflake.cli.api.cli_global_context import span from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.project.util import to_identifier from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.utils.path_utils import delete diff --git a/src/snowflake/cli/_plugins/nativeapp/bundle_context.py b/src/snowflake/cli/_plugins/nativeapp/bundle_context.py index a8e6427eda..a680ff8779 100644 --- a/src/snowflake/cli/_plugins/nativeapp/bundle_context.py +++ b/src/snowflake/cli/_plugins/nativeapp/bundle_context.py @@ -18,7 +18,7 @@ List, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping @dataclass diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py index 922fdc55ab..02dc4ff11a 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py @@ -20,7 +20,7 @@ from click import ClickException from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( +from snowflake.cli.api.project.schemas.entities.common import ( PathMapping, ProcessorMapping, ) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py index 7681f1fab3..c8f675663d 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py @@ -36,9 +36,7 @@ from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.metrics import CLICounterField -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( - ProcessorMapping, -) +from snowflake.cli.api.project.schemas.entities.common import ProcessorMapping ProcessorClassType = type[ArtifactProcessor] diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py index c62743eab0..04290d1f8d 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py @@ -39,7 +39,7 @@ from snowflake.cli._plugins.stage.diff import to_stage_path from snowflake.cli.api.artifacts.bundle_map import BundleMap from snowflake.cli.api.console import cli_console as cc -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( +from snowflake.cli.api.project.schemas.entities.common import ( PathMapping, ProcessorMapping, ) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py index 2fc58cd4ab..1396665e92 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py @@ -51,7 +51,7 @@ from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.metrics import CLICounterField -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( +from snowflake.cli.api.project.schemas.entities.common import ( PathMapping, ProcessorMapping, ) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index ff1ee55084..95d8ca4d93 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -26,7 +26,7 @@ from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.metrics import CLICounterField -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( +from snowflake.cli.api.project.schemas.entities.common import ( PathMapping, ProcessorMapping, ) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index bc6e2805a2..353b40effc 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -79,9 +79,8 @@ ) from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError -from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.entities.common import ( - EntityModelBase, + ArtifactsBaseModel, Identifier, PostDeployHook, ) @@ -91,7 +90,6 @@ UpdatableModel, ) from snowflake.cli.api.project.schemas.v1.native_app.package import DistributionOptions -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.project.util import ( SCHEMA_AND_NAME, VALID_IDENTIFIER_REGEX, @@ -146,19 +144,12 @@ class ApplicationPackageChildField(UpdatableModel): ) -class ApplicationPackageEntityModel(EntityModelBase): +class ApplicationPackageEntityModel(ArtifactsBaseModel): type: Literal["application package"] = DiscriminatorField() # noqa: A003 - artifacts: Artifacts = Field( - title="List of paths or file source/destination pairs to add to the deploy root", - ) bundle_root: Optional[str] = Field( title="Folder at the root of your project where artifacts necessary to perform the bundle step are stored", default="output/bundle/", ) - deploy_root: Optional[str] = Field( - title="Folder at the root of your project where the build step copies the artifacts", - default="output/deploy/", - ) children_artifacts_dir: Optional[str] = Field( title="Folder under deploy_root where the child artifacts will be stored", default="_children/", @@ -210,21 +201,6 @@ def append_test_resource_suffix_to_identifier( return input_value.model_copy(update=dict(name=with_suffix)) return with_suffix - @field_validator("artifacts") - @classmethod - def transform_artifacts(cls, orig_artifacts: Artifacts) -> List[PathMapping]: - transformed_artifacts: List[PathMapping] = [] - if orig_artifacts is None: - return transformed_artifacts - - for artifact in orig_artifacts: - if isinstance(artifact, PathMapping): - transformed_artifacts.append(artifact) - else: - transformed_artifacts.append(PathMapping(src=artifact)) - - return transformed_artifacts - @field_validator("stage") @classmethod def validate_source_stage(cls, input_value: str): diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py index 25e73eda2a..28db01d068 100644 --- a/src/snowflake/cli/_plugins/snowpark/commands.py +++ b/src/snowflake/cli/_plugins/snowpark/commands.py @@ -94,11 +94,11 @@ from snowflake.cli.api.project.definition_conversion import ( convert_project_definition_to_v2, ) +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.project.schemas.project_definition import ( ProjectDefinition, ProjectDefinitionV2, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.secure_path import SecurePath from snowflake.connector import DictCursor, ProgrammingError from snowflake.connector.cursor import SnowflakeCursor diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py b/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py index 5b355b804a..236742b7e4 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py @@ -20,16 +20,16 @@ from pydantic import Field, field_validator from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.entities.common import ( + Artifacts, EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel, + PathMapping, ) from snowflake.cli.api.project.schemas.updatable_model import ( DiscriminatorField, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.project.schemas.v1.snowpark.argument import Argument diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py index cbc05e2c2d..6e103d840e 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py @@ -20,13 +20,13 @@ from pathlib import Path, PurePosixPath from typing import Optional -from snowflake.cli._plugins.snowpark.snowpark_entity_model import PathMapping from snowflake.cli._plugins.snowpark.zipper import zip_dir from snowflake.cli.api.console import cli_console from snowflake.cli.api.constants import DEPLOYMENT_STAGE from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.project.project_paths import ProjectPaths +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.secure_path import SecurePath diff --git a/src/snowflake/cli/_plugins/streamlit/manager.py b/src/snowflake/cli/_plugins/streamlit/manager.py index a83f553df2..4b677c903e 100644 --- a/src/snowflake/cli/_plugins/streamlit/manager.py +++ b/src/snowflake/cli/_plugins/streamlit/manager.py @@ -40,7 +40,7 @@ from snowflake.cli.api.console import cli_console from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor from snowflake.connector.errors import ProgrammingError diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py b/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py index 0ffcf8f124..f0e6a8c43e 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py @@ -16,16 +16,16 @@ from typing import Literal, Optional from pydantic import Field, field_validator -from snowflake.cli.api.project.schemas.commons import Artifacts from snowflake.cli.api.project.schemas.entities.common import ( + Artifacts, EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel, + PathMapping, ) from snowflake.cli.api.project.schemas.updatable_model import ( DiscriminatorField, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping class StreamlitEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel): diff --git a/src/snowflake/cli/api/artifacts/bundle_map.py b/src/snowflake/cli/api/artifacts/bundle_map.py index 93089f71e2..b60ac9e501 100644 --- a/src/snowflake/cli/api/artifacts/bundle_map.py +++ b/src/snowflake/cli/api/artifacts/bundle_map.py @@ -11,7 +11,7 @@ SourceNotFoundError, TooManyFilesError, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.utils.path_utils import resolve_without_follow ArtifactPredicate = Callable[[Path, Path], bool] diff --git a/src/snowflake/cli/api/project/schemas/commons.py b/src/snowflake/cli/api/project/schemas/commons.py deleted file mode 100644 index b512afbb39..0000000000 --- a/src/snowflake/cli/api/project/schemas/commons.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import List, Union - -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping - -Artifacts = List[Union[PathMapping, str]] diff --git a/src/snowflake/cli/api/project/schemas/entities/common.py b/src/snowflake/cli/api/project/schemas/entities/common.py index d9036d9a4c..3031cb4126 100644 --- a/src/snowflake/cli/api/project/schemas/entities/common.py +++ b/src/snowflake/cli/api/project/schemas/entities/common.py @@ -15,7 +15,7 @@ from __future__ import annotations from abc import ABC -from typing import Dict, Generic, List, Optional, TypeVar, Union +from typing import Any, Dict, Generic, List, Optional, TypeVar, Union from pydantic import Field, PrivateAttr, field_validator from snowflake.cli.api.identifiers import FQN @@ -162,3 +162,76 @@ def get_secrets_sql(self) -> str | None: return None secrets = ", ".join(f"'{key}'={value}" for key, value in self.secrets.items()) return f"secrets=({secrets})" + + +class ProcessorMapping(UpdatableModel): + name: str = Field( + title="Name of a processor to invoke on a collection of artifacts." + ) + properties: Optional[Dict[str, Any]] = Field( + title="A set of key-value pairs used to configure the output of the processor. Consult a specific processor's documentation for more details on the supported properties.", + default=None, + ) + + +class PathMapping(UpdatableModel): + src: str = Field( + title="Source path or glob pattern (relative to project root)", default=None + ) + + dest: Optional[str] = Field( + title="Destination path on stage", + description="Paths are relative to stage root; paths ending with a slash indicate that the destination is a directory which source files should be copied into.", + default=None, + ) + + processors: Optional[List[Union[str, ProcessorMapping]]] = Field( + title="List of processors to apply to matching source files during bundling.", + default=[], + ) + + @field_validator("processors") + @classmethod + def transform_processors( + cls, input_values: Optional[List[Union[str, Dict, ProcessorMapping]]] + ) -> List[ProcessorMapping]: + if input_values is None: + return [] + + transformed_processors: List[ProcessorMapping] = [] + for input_processor in input_values: + if isinstance(input_processor, str): + transformed_processors.append(ProcessorMapping(name=input_processor)) + elif isinstance(input_processor, Dict): + transformed_processors.append(ProcessorMapping(**input_processor)) + else: + transformed_processors.append(input_processor) + return transformed_processors + + +Artifacts = List[Union[PathMapping, str]] + + +class ArtifactsBaseModel(EntityModelBase): + artifacts: Artifacts = Field( + title="List of paths or file source/destination pairs to add to the deploy root", + ) + deploy_root: Optional[str] = Field( + title="Folder at the root of your project where the build step copies the artifacts", + default="output/deploy/", + ) + + @field_validator("artifacts") + @classmethod + def transform_artifacts(cls, orig_artifacts: Artifacts) -> List[PathMapping]: + transformed_artifacts: List[PathMapping] = [] + if orig_artifacts is None: + return transformed_artifacts + + for artifact in orig_artifacts: + if isinstance(artifact, PathMapping): + transformed_artifacts.append(artifact) + else: + transformed_artifacts.append(PathMapping(src=artifact)) + + return transformed_artifacts diff --git a/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py b/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py index d0c065c7f3..f1cab68139 100644 --- a/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py +++ b/src/snowflake/cli/api/project/schemas/v1/native_app/native_app.py @@ -18,14 +18,13 @@ from typing import List, Optional from pydantic import Field, field_validator -from snowflake.cli.api.project.schemas.commons import Artifacts +from snowflake.cli.api.project.schemas.entities.common import Artifacts, PathMapping from snowflake.cli.api.project.schemas.updatable_model import UpdatableModel from snowflake.cli.api.project.schemas.v1.native_app.application import ( Application, ApplicationV11, ) from snowflake.cli.api.project.schemas.v1.native_app.package import Package, PackageV11 -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from snowflake.cli.api.project.util import ( SCHEMA_AND_NAME, ) diff --git a/src/snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py b/src/snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py deleted file mode 100644 index ba9eaed997..0000000000 --- a/src/snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2024 Snowflake Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Any, Dict, List, Optional, Union - -from pydantic import Field, field_validator -from snowflake.cli.api.project.schemas.updatable_model import UpdatableModel - - -class ProcessorMapping(UpdatableModel): - name: str = Field( - title="Name of a processor to invoke on a collection of artifacts." - ) - properties: Optional[Dict[str, Any]] = Field( - title="A set of key-value pairs used to configure the output of the processor. Consult a specific processor's documentation for more details on the supported properties.", - default=None, - ) - - -class PathMapping(UpdatableModel): - src: str = Field( - title="Source path or glob pattern (relative to project root)", default=None - ) - - dest: Optional[str] = Field( - title="Destination path on stage", - description="Paths are relative to stage root; paths ending with a slash indicate that the destination is a directory which source files should be copied into.", - default=None, - ) - - processors: Optional[List[Union[str, ProcessorMapping]]] = Field( - title="List of processors to apply to matching source files during bundling.", - default=[], - ) - - @field_validator("processors") - @classmethod - def transform_processors( - cls, input_values: Optional[List[Union[str, Dict, ProcessorMapping]]] - ) -> List[ProcessorMapping]: - if input_values is None: - return [] - - transformed_processors: List[ProcessorMapping] = [] - for input_processor in input_values: - if isinstance(input_processor, str): - transformed_processors.append(ProcessorMapping(name=input_processor)) - elif isinstance(input_processor, Dict): - transformed_processors.append(ProcessorMapping(**input_processor)) - else: - transformed_processors.append(input_processor) - return transformed_processors diff --git a/tests/api/artifacts/test_bundle_map.py b/tests/api/artifacts/test_bundle_map.py index c74f319121..92964a5e69 100644 --- a/tests/api/artifacts/test_bundle_map.py +++ b/tests/api/artifacts/test_bundle_map.py @@ -27,7 +27,7 @@ SourceNotFoundError, TooManyFilesError, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.utils.path_utils import resolve_without_follow from tests.nativeapp.utils import touch diff --git a/tests/helpers/__snapshots__/test_v1_to_v2.ambr b/tests/helpers/__snapshots__/test_v1_to_v2.ambr index ddfd878c1b..00c92bd693 100644 --- a/tests/helpers/__snapshots__/test_v1_to_v2.ambr +++ b/tests/helpers/__snapshots__/test_v1_to_v2.ambr @@ -232,8 +232,8 @@ meta: role: pkg_role identifier: <% fn.concat_ids('myapp', '_pkg_', fn.sanitize_id(fn.get_username('unknown_user')) | lower) %> - type: application package artifacts: [] + type: application package app: identifier: myapp_app type: application @@ -300,7 +300,6 @@ post_deploy: - sql_script: scripts/post_pkg_deploy.sql identifier: my_app_package - type: application package artifacts: - src: app/* dest: ./ @@ -311,8 +310,9 @@ - name: templates properties: foo: bar - bundle_root: my_output/my_bundle deploy_root: my_output/my_deploy + type: application package + bundle_root: my_output/my_bundle generated_root: __my_generated_files stage: app_src.my_stage scratch_stage: app_src.my_scratch @@ -432,10 +432,10 @@ meta: role: pkg_role identifier: <% fn.concat_ids('myapp', '_pkg_', fn.sanitize_id(fn.get_username('unknown_user')) | lower) %> - type: application package artifacts: - src: app/* dest: ./ + type: application package app: identifier: myapp_app type: application diff --git a/tests/nativeapp/codegen/snowpark/test_python_processor.py b/tests/nativeapp/codegen/snowpark/test_python_processor.py index 045dea8c10..27adfeb1d8 100644 --- a/tests/nativeapp/codegen/snowpark/test_python_processor.py +++ b/tests/nativeapp/codegen/snowpark/test_python_processor.py @@ -37,9 +37,7 @@ from snowflake.cli._plugins.nativeapp.entities.application_package import ( ApplicationPackageEntityModel, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( - ProcessorMapping, -) +from snowflake.cli.api.project.schemas.entities.common import ProcessorMapping from tests.nativeapp.utils import assert_dir_snapshot from tests.testing_utils.files_and_dirs import pushd, temp_local_dir diff --git a/tests/nativeapp/codegen/templating/test_templates_processor.py b/tests/nativeapp/codegen/templating/test_templates_processor.py index 65eb5b3dac..5d3b66403d 100644 --- a/tests/nativeapp/codegen/templating/test_templates_processor.py +++ b/tests/nativeapp/codegen/templating/test_templates_processor.py @@ -26,7 +26,7 @@ ) from snowflake.cli._plugins.nativeapp.exceptions import InvalidTemplateInFileError from snowflake.cli.api.exceptions import InvalidTemplate -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping from tests.nativeapp.utils import ( CLI_GLOBAL_TEMPLATE_CONTEXT, diff --git a/tests/nativeapp/test_annotation_processor_config.py b/tests/nativeapp/test_annotation_processor_config.py index 24442b3efd..016565d1b4 100644 --- a/tests/nativeapp/test_annotation_processor_config.py +++ b/tests/nativeapp/test_annotation_processor_config.py @@ -14,9 +14,7 @@ import pytest from snowflake.cli.api.project.definition import load_project -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( - ProcessorMapping, -) +from snowflake.cli.api.project.schemas.entities.common import ProcessorMapping @pytest.mark.parametrize( diff --git a/tests/nativeapp/test_artifacts.py b/tests/nativeapp/test_artifacts.py index b0372d2e2b..b098e987ce 100644 --- a/tests/nativeapp/test_artifacts.py +++ b/tests/nativeapp/test_artifacts.py @@ -32,7 +32,7 @@ VersionInfo, ) from snowflake.cli.api.project.definition import load_project -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.project.util import to_identifier from yaml import safe_dump diff --git a/tests/project/test_config.py b/tests/project/test_config.py index aa9e63834d..a1b809413c 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -20,10 +20,10 @@ import pytest from snowflake.cli.api.project.definition import load_project from snowflake.cli.api.project.errors import SchemaValidationError +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.project.schemas.project_definition import ( build_project_definition, ) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping from tests.nativeapp.factories import PdfV10Factory diff --git a/tests/project/test_project_definition_v2.py b/tests/project/test_project_definition_v2.py index e639f41cfa..614b2c046e 100644 --- a/tests/project/test_project_definition_v2.py +++ b/tests/project/test_project_definition_v2.py @@ -15,7 +15,6 @@ import pytest from snowflake.cli._plugins.snowpark.snowpark_entity_model import ( - PathMapping, SnowparkEntityModel, ) from snowflake.cli.api.project.definition_conversion import ( @@ -23,6 +22,7 @@ ) from snowflake.cli.api.project.definition_manager import DefinitionManager from snowflake.cli.api.project.errors import SchemaValidationError +from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.project.schemas.entities.entities import ( ALL_ENTITIES, ALL_ENTITY_MODELS, From 1fe4e7ce04460cc31bb9e0ac1c9dcf6ec7e254fc Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Mon, 9 Dec 2024 04:05:19 -0500 Subject: [PATCH 05/10] Fix rebase issues in feature/unification-artifacts (#1932) Fix feature branch gates --- tests/nativeapp/test_artifacts.py | 2 +- tests/snowpark/test_function.py | 18 ++++++++++++++++-- tests/streamlit/test_artifacts.py | 4 ++-- tests/streamlit/test_commands.py | 29 ----------------------------- tests/testing_utils/fixtures.py | 3 +++ tests_integration/conftest.py | 3 +++ 6 files changed, 25 insertions(+), 34 deletions(-) diff --git a/tests/nativeapp/test_artifacts.py b/tests/nativeapp/test_artifacts.py index b098e987ce..6cec78e676 100644 --- a/tests/nativeapp/test_artifacts.py +++ b/tests/nativeapp/test_artifacts.py @@ -19,6 +19,7 @@ import pytest from click import ClickException from snowflake.cli._plugins.nativeapp.artifacts import ( + VersionInfo, build_bundle, find_events_definitions_in_manifest_file, find_version_info_in_manifest_file, @@ -29,7 +30,6 @@ NotInDeployRootError, SourceNotFoundError, TooManyFilesError, - VersionInfo, ) from snowflake.cli.api.project.definition import load_project from snowflake.cli.api.project.schemas.entities.common import PathMapping diff --git a/tests/snowpark/test_function.py b/tests/snowpark/test_function.py index a2887da832..b071277602 100644 --- a/tests/snowpark/test_function.py +++ b/tests/snowpark/test_function.py @@ -249,7 +249,14 @@ def test_deploy_function_no_changes( ) @mock_session_has_warehouse def test_deploy_function_needs_update_because_packages_changes( - mock_download, mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name, enable_snowpark_glob_support_feature_flag + mock_download, + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, + project_name, + enable_snowpark_glob_support_feature_flag, ): mock_download.return_value = DownloadUnavailablePackagesResult() rows = [ @@ -304,7 +311,14 @@ def test_deploy_function_needs_update_because_packages_changes( ) @mock_session_has_warehouse def test_deploy_function_needs_update_because_handler_changes( - mock_download, mock_connector, runner, mock_ctx, mock_cursor, project_directory, project_name, enable_snowpark_glob_support_feature_flag + mock_download, + mock_connector, + runner, + mock_ctx, + mock_cursor, + project_directory, + project_name, + enable_snowpark_glob_support_feature_flag, ): mock_download.return_value = DownloadUnavailablePackagesResult() rows = [ diff --git a/tests/streamlit/test_artifacts.py b/tests/streamlit/test_artifacts.py index 4b75406d1c..d63f3033df 100644 --- a/tests/streamlit/test_artifacts.py +++ b/tests/streamlit/test_artifacts.py @@ -112,7 +112,7 @@ @mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put") @mock.patch( "snowflake.cli._plugins.connection.util.get_ui_parameters", - return_value={UIParameter.NA_ENABLE_REGIONLESS_REDIRECT: "false"}, + return_value={UIParameter.NA_ENABLE_REGIONLESS_REDIRECT: False}, ) def test_deploy_with_artifacts( mock_param, @@ -252,7 +252,7 @@ def test_deploy_with_artifacts( @mock.patch("snowflake.cli._plugins.snowpark.commands.StageManager.put") @mock.patch( "snowflake.cli._plugins.connection.util.get_ui_parameters", - return_value={UIParameter.NA_ENABLE_REGIONLESS_REDIRECT: "false"}, + return_value={UIParameter.NA_ENABLE_REGIONLESS_REDIRECT: False}, ) def test_deploy_with_artifacts_from_other_directory( mock_param, diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index cd39056335..9a859ad030 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -920,35 +920,6 @@ def test_deploy_streamlit_main_and_pages_files_experimental_replace( ] -@pytest.mark.parametrize( - "project_name,opts", - [ - ("example_streamlit", {"streamlit": {"pages_dir": "foo.bar"}}), - ("example_streamlit", {"streamlit": {"env_file": "foo.bar"}}), - ( - "example_streamlit_v2", - {"entities": {"test_streamlit": {"pages_dir": "foo.bar"}}}, - ), - ( - "example_streamlit_v2", - {"entities": {"test_streamlit": {"artifacts": ["foo.bar"]}}}, - ), - ], -) -@mock.patch("snowflake.connector.connect") -def test_deploy_streamlit_nonexisting_file( - mock_connector, runner, mock_ctx, snapshot, project_directory, opts, project_name -): - ctx = mock_ctx() - mock_connector.return_value = ctx - - with project_directory(project_name, merge_project_definition=opts): - result = runner.invoke(["streamlit", "deploy"]) - - assert result.exit_code == 1 - assert result.output == snapshot - - @mock.patch("snowflake.connector.connect") def test_share_streamlit(mock_connector, runner, mock_ctx): ctx = mock_ctx() diff --git a/tests/testing_utils/fixtures.py b/tests/testing_utils/fixtures.py index 457acec1c6..192776fcc9 100644 --- a/tests/testing_utils/fixtures.py +++ b/tests/testing_utils/fixtures.py @@ -473,5 +473,8 @@ def enable_snowpark_glob_support_feature_flag(): with mock.patch( f"snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled", return_value=True, + ), mock.patch( + f"snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_disabled", + return_value=False, ): yield diff --git a/tests_integration/conftest.py b/tests_integration/conftest.py index 6be62f1ce5..1ac9bda801 100644 --- a/tests_integration/conftest.py +++ b/tests_integration/conftest.py @@ -266,5 +266,8 @@ def enable_snowpark_glob_support_feature_flag(): with mock.patch( f"snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled", return_value=True, + ), mock.patch( + f"snowflake.cli.api.feature_flags.FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_disabled", + return_value=False, ): yield From 40354fbecbc947fac0f8ba40fa6f0ff359e25389 Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Mon, 9 Dec 2024 13:15:30 -0800 Subject: [PATCH 06/10] Fixes to imports following rebase --- tests/nativeapp/codegen/test_compiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/nativeapp/codegen/test_compiler.py b/tests/nativeapp/codegen/test_compiler.py index 7da3382588..6f703e3d9e 100644 --- a/tests/nativeapp/codegen/test_compiler.py +++ b/tests/nativeapp/codegen/test_compiler.py @@ -28,13 +28,13 @@ from snowflake.cli._plugins.nativeapp.entities.application_package import ( ApplicationPackageEntityModel, ) -from snowflake.cli.api.project.schemas.project_definition import ( - build_project_definition, -) -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( +from snowflake.cli.api.project.schemas.entities.common import ( PathMapping, ProcessorMapping, ) +from snowflake.cli.api.project.schemas.project_definition import ( + build_project_definition, +) @pytest.fixture() From bf1a0e2d25ec47ab6d9dd008fd6a85bf7c66e52b Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Wed, 11 Dec 2024 11:45:44 +0100 Subject: [PATCH 07/10] =?UTF-8?q?Moved=20Snowpark=20and=20Streamlit=20arti?= =?UTF-8?q?facts=20to=20separate=20directory=20in=20outpu=E2=80=A6=20(#190?= =?UTF-8?q?9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved Snowpark and Streamlit artifacts to separate directory in output/deploy --- .../cli/_plugins/snowpark/commands.py | 10 +- .../snowpark/snowpark_project_paths.py | 31 ++++-- .../cli/_plugins/streamlit/manager.py | 10 +- .../streamlit/streamlit_project_paths.py | 7 +- .../cli/api/project/project_paths.py | 16 ++- tests/snowpark/test_artifacts.py | 42 ++++---- tests/snowpark/test_build.py | 4 +- tests/snowpark/test_function.py | 25 ++++- tests/snowpark/test_procedure.py | 19 +++- tests/snowpark/test_project_paths.py | 79 ++++++++------ tests/streamlit/test_artifacts.py | 100 +++++++++++------- tests/streamlit/test_commands.py | 2 +- tests_integration/test_snowpark.py | 23 ++-- .../testing_utils/snowpark_utils.py | 4 +- 14 files changed, 242 insertions(+), 130 deletions(-) diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py index 28db01d068..37f093a930 100644 --- a/src/snowflake/cli/_plugins/snowpark/commands.py +++ b/src/snowflake/cli/_plugins/snowpark/commands.py @@ -220,19 +220,19 @@ def build_artifacts_mappings( ) -> Tuple[EntityToImportPathsMapping, StageToArtefactMapping]: stages_to_artifact_map: StageToArtefactMapping = defaultdict(set) entities_to_imports_map: EntityToImportPathsMapping = defaultdict(set) - for entity_id, entity in snowpark_entities.items(): + for name, entity in snowpark_entities.items(): stage = entity.stage required_artifacts = set() for artefact in entity.artifacts: artefact_dto = project_paths.get_artefact_dto(artefact) required_artifacts.add(artefact_dto) - entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage)) + entities_to_imports_map[name].add(artefact_dto.import_path(stage)) stages_to_artifact_map[stage].update(required_artifacts) deps_artefact = project_paths.get_dependencies_artefact() if deps_artefact.post_build_path.exists(): stages_to_artifact_map[stage].add(deps_artefact) - entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage)) + entities_to_imports_map[name].add(deps_artefact.import_path(stage)) return entities_to_imports_map, stages_to_artifact_map @@ -330,7 +330,7 @@ def build( anaconda_packages_manager = AnacondaPackagesManager() # Clean up deploy root - project_paths.remove_up_deploy_root() + project_paths.remove_up_bundle_root() # Resolve dependencies if project_paths.requirements.exists(): @@ -389,7 +389,7 @@ def build( for artefact in artifacts: bundle_map = BundleMap( project_root=artefact.project_root, - deploy_root=project_paths.deploy_root, + deploy_root=project_paths.bundle_root, ) bundle_map.add(PathMapping(src=str(artefact.path), dest=artefact.dest)) diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py index 6e103d840e..2bfe957cf2 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_project_paths.py @@ -25,7 +25,7 @@ from snowflake.cli.api.constants import DEPLOYMENT_STAGE from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.project_paths import ProjectPaths +from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root from snowflake.cli.api.project.schemas.entities.common import PathMapping from snowflake.cli.api.secure_path import SecurePath @@ -45,6 +45,7 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): return Artefact( project_root=self.project_root, + bundle_root=self.bundle_root, dest=artifact_path.dest, path=Path(artifact_path.src), ) @@ -57,7 +58,10 @@ def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact: def get_dependencies_artefact(self) -> Artefact: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): return Artefact( - project_root=self.project_root, dest=None, path=Path("dependencies.zip") + project_root=self.project_root, + bundle_root=self.bundle_root, + dest=None, + path=Path("dependencies.zip"), ) else: return ArtefactOldBuild( @@ -74,19 +78,29 @@ def snowflake_requirements(self) -> SecurePath: def requirements(self) -> SecurePath: return SecurePath(self.path_relative_to_root(Path("requirements.txt"))) + @property + def bundle_root(self) -> Path: + return bundle_root(self.project_root, "snowpark") + @dataclass(unsafe_hash=True) class Artefact: """Helper for getting paths related to given artefact.""" project_root: Path + bundle_root: Path path: Path dest: str | None = None def __init__( - self, project_root: Path, path: Path, dest: Optional[str] = None + self, + project_root: Path, + bundle_root: Path, + path: Path, + dest: Optional[str] = None, ) -> None: self.project_root = project_root + self.bundle_root = bundle_root self.path = path self.dest = dest if self.dest and not self._is_dest_a_file() and not self.dest.endswith("/"): @@ -117,15 +131,15 @@ def post_build_path(self) -> Path: """ Returns post-build artefact path. Directories are mapped to corresponding .zip files. """ - deploy_root = self.deploy_root() + bundle_root = self.bundle_root path = ( self._path_until_asterisk() if glob.has_magic(str(self.path)) else self.path.parent ) if self._is_dest_a_file(): - return deploy_root / self.dest # type: ignore - return deploy_root / (self.dest or path) / self._artefact_name + return bundle_root / self.dest # type: ignore + return bundle_root / (self.dest or path) / self._artefact_name def upload_path(self, stage: FQN | str | None) -> str: """ @@ -153,9 +167,6 @@ def import_path(self, stage: FQN | str | None) -> str: """Path for UDF/sproc imports clause.""" return self.upload_path(stage) + self._artefact_name - def deploy_root(self) -> Path: - return self.project_root / "output" - def _is_dest_a_file(self) -> bool: if not self.dest: return False @@ -183,7 +194,7 @@ class ArtefactOldBuild(Artefact): dest: str | None = None def __init__(self, path: Path, dest: Optional[str] = None) -> None: - super().__init__(project_root=Path(), path=path, dest=dest) + super().__init__(project_root=Path(), bundle_root=Path(), path=path, dest=dest) @property def _artefact_name(self) -> str: diff --git a/src/snowflake/cli/_plugins/streamlit/manager.py b/src/snowflake/cli/_plugins/streamlit/manager.py index 4b677c903e..f6388a3f8f 100644 --- a/src/snowflake/cli/_plugins/streamlit/manager.py +++ b/src/snowflake/cli/_plugins/streamlit/manager.py @@ -68,29 +68,31 @@ def _put_streamlit_files( if not artifacts: return stage_manager = StageManager() + # We treat the bundle root as deploy root bundle_map = BundleMap( project_root=streamlit_project_paths.project_root, - deploy_root=streamlit_project_paths.deploy_root, + deploy_root=streamlit_project_paths.bundle_root, ) for artifact in artifacts: bundle_map.add(PathMapping(src=str(artifact.src), dest=artifact.dest)) # Clean up deploy root - streamlit_project_paths.remove_up_deploy_root() + streamlit_project_paths.remove_up_bundle_root() for (absolute_src, absolute_dest) in bundle_map.all_mappings( absolute=True, expand_directories=True ): if absolute_src.is_file(): + # We treat the bundle root as deploy root symlink_or_copy( absolute_src, absolute_dest, - deploy_root=streamlit_project_paths.deploy_root, + deploy_root=streamlit_project_paths.bundle_root, ) # Temporary solution, will be replaced with diff stage_path = ( PurePosixPath(absolute_dest) - .relative_to(streamlit_project_paths.deploy_root) + .relative_to(streamlit_project_paths.bundle_root) .parent ) full_stage_path = f"{stage_root}/{stage_path}".rstrip("/") diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py index fbe89959b2..df55dddc8e 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_project_paths.py @@ -14,8 +14,9 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path -from snowflake.cli.api.project.project_paths import ProjectPaths +from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root @dataclass @@ -23,3 +24,7 @@ class StreamlitProjectPaths(ProjectPaths): """ This class allows you to manage files paths related to given project. """ + + @property + def bundle_root(self) -> Path: + return bundle_root(self.project_root, "streamlit") diff --git a/src/snowflake/cli/api/project/project_paths.py b/src/snowflake/cli/api/project/project_paths.py index f6b2a14d99..544fa36dbf 100644 --- a/src/snowflake/cli/api/project/project_paths.py +++ b/src/snowflake/cli/api/project/project_paths.py @@ -8,9 +8,15 @@ class ProjectPaths: project_root: Path @property - def deploy_root(self) -> Path: - return self.project_root / "output" + def bundle_root(self) -> Path: + return bundle_root(self.project_root) - def remove_up_deploy_root(self) -> None: - if self.deploy_root.exists(): - rmtree(self.deploy_root) + def remove_up_bundle_root(self) -> None: + if self.bundle_root.exists(): + rmtree(self.bundle_root) + + +def bundle_root(root: Path, app_type: str | None = None) -> Path: + if app_type: + return root / "output" / "bundle" / app_type + return root / "output" / "bundle" diff --git a/tests/snowpark/test_artifacts.py b/tests/snowpark/test_artifacts.py index c573ab9b6d..fe16a06119 100644 --- a/tests/snowpark/test_artifacts.py +++ b/tests/snowpark/test_artifacts.py @@ -12,42 +12,44 @@ lambda _: True, ) +bundle_root = Path("output") / "bundle" / "snowpark" + @pytest.mark.parametrize( "artifacts, local_path, stage_path", [ - ("src", Path("output") / "src.zip", "/"), - ("src/", Path("output") / "src.zip", "/"), - ("src/*", Path("output") / "src.zip", "/"), - ("src/*.py", Path("output") / "src.zip", "/"), + ("src", bundle_root / "src.zip", "/"), + ("src/", bundle_root / "src.zip", "/"), + ("src/*", bundle_root / "src.zip", "/"), + ("src/*.py", bundle_root / "src.zip", "/"), ( "src/dir/dir_app.py", - Path("output") / "src" / "dir" / "dir_app.py", + bundle_root / "src" / "dir" / "dir_app.py", "/src/dir/", ), ( {"src": "src/**/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, - "output/source/dir/apps/dir_app.py", + bundle_root / "source" / "dir" / "apps" / "dir_app.py", "/source/dir/apps/", ), ], @@ -111,38 +113,38 @@ def test_build_and_deploy_with_artifacts( @pytest.mark.parametrize( "artifact, local_path, stage_path", [ - ("src", Path("output") / "src.zip", "/"), - ("src/", Path("output") / "src.zip", "/"), - ("src/*", Path("output") / "src.zip", "/"), - ("src/*.py", Path("output") / "src.zip", "/"), + ("src", bundle_root / "src.zip", "/"), + ("src/", bundle_root / "src.zip", "/"), + ("src/*", bundle_root / "src.zip", "/"), + ("src/*.py", bundle_root / "src.zip", "/"), ( "src/dir/dir_app.py", - Path("output") / "src" / "dir" / "dir_app.py", + bundle_root / "src" / "dir" / "dir_app.py", "/src/dir/", ), ( {"src": "src/**/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/*", "dest": "source/"}, - Path("output") / "source" / "src.zip", + bundle_root / "source" / "src.zip", "/source/", ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, - Path("output") / "source" / "dir" / "apps" / "dir_app.py", + bundle_root / "source" / "dir" / "apps" / "dir_app.py", "/source/dir/apps/", ), ], diff --git a/tests/snowpark/test_build.py b/tests/snowpark/test_build.py index 9298ed2b32..fb7c4841fc 100644 --- a/tests/snowpark/test_build.py +++ b/tests/snowpark/test_build.py @@ -58,7 +58,9 @@ def test_build_with_glob_patterns_in_artifacts( result = runner.invoke(["snowpark", "build", "--ignore-anaconda"]) assert result.exit_code == 0, result.output - _assert_zip_contains(tmp_dir / "output" / zip_name, expected_files) + _assert_zip_contains( + tmp_dir / "output" / "bundle" / "snowpark" / zip_name, expected_files + ) def _assert_zip_contains(app_zip: str, expected_files: Set[str]): diff --git a/tests/snowpark/test_function.py b/tests/snowpark/test_function.py index b071277602..741993c55a 100644 --- a/tests/snowpark/test_function.py +++ b/tests/snowpark/test_function.py @@ -236,7 +236,11 @@ def test_deploy_function_no_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), ] @@ -286,7 +290,11 @@ def test_deploy_function_needs_update_because_packages_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) @@ -348,8 +356,11 @@ def test_deploy_function_needs_update_because_handler_changes( ] assert queries == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" - f" auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace function IDENTIFIER('MockDatabase.MockSchema.func1')(a string default 'default value', b variant) @@ -712,3 +723,9 @@ def test_command_aliases(mock_connector, runner, mock_ctx, command, parameters): queries = ctx.get_queries() assert queries[0] == queries[1] + + +def _put_query(project_root: Path, source: str, dest: str): + return dedent( + f"put file://{project_root.resolve() / 'output' / 'bundle' / 'snowpark' / source} {dest} auto_compress=false parallel=4 overwrite=True" + ) diff --git a/tests/snowpark/test_procedure.py b/tests/snowpark/test_procedure.py index 3b2069e7fd..e00152da3d 100644 --- a/tests/snowpark/test_procedure.py +++ b/tests/snowpark/test_procedure.py @@ -109,7 +109,11 @@ def test_deploy_procedure( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(tmp).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/ auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(tmp), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) @@ -199,8 +203,11 @@ def test_deploy_procedure_with_external_access( ) assert ctx.get_queries() == [ "create stage if not exists IDENTIFIER('MockDatabase.MockSchema.dev_deployment') comment='deployments managed by Snowflake CLI'", - f"put file://{Path(project_dir).resolve()}/output/my_snowpark_project/app.py @MockDatabase.MockSchema.dev_deployment/my_snowpark_project/" - f" auto_compress=false parallel=4 overwrite=True", + _put_query( + Path(project_dir), + "my_snowpark_project/app.py", + "@MockDatabase.MockSchema.dev_deployment/my_snowpark_project/", + ), dedent( """\ create or replace procedure IDENTIFIER('MockDatabase.MockSchema.procedureName')(name string) @@ -835,3 +842,9 @@ def test_snowpark_fail_if_no_active_warehouse(runner, mock_ctx, project_director "The command requires warehouse. No warehouse found in current connection." in result.output ) + + +def _put_query(project_root: Path, source: str, dest: str): + return dedent( + f"put file://{project_root.resolve() / 'output' / 'bundle' / 'snowpark' / source} {dest} auto_compress=false parallel=4 overwrite=True" + ) diff --git a/tests/snowpark/test_project_paths.py b/tests/snowpark/test_project_paths.py index 55796e1a98..19618424ac 100644 --- a/tests/snowpark/test_project_paths.py +++ b/tests/snowpark/test_project_paths.py @@ -4,6 +4,9 @@ import pytest from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact +bundle_root = Path("output") / "bundle" / "snowpark" +absolute_bundle_root = Path.cwd().absolute() / "output" / "bundle" / "snowpark" + @pytest.mark.parametrize( "path, dest, is_file, expected_path", @@ -31,7 +34,7 @@ def test_artifact_import_path(mock_ctx_context, path, dest, is_file, expected_pa stage = "stage" with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - import_path = Artefact(Path(), Path(path), dest).import_path(stage) + import_path = Artefact(Path(), bundle_root, Path(path), dest).import_path(stage) assert import_path == expected_path @@ -61,7 +64,9 @@ def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_pa mock_ctx_context.return_value.connection = mock_connection with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - upload_path = Artefact(Path(), Path(path), dest).upload_path("stage") + upload_path = Artefact(Path(), bundle_root, Path(path), dest).upload_path( + "stage" + ) assert upload_path == expected_path @@ -69,39 +74,41 @@ def test_artifact_upload_path(mock_ctx_context, path, dest, is_file, expected_pa @pytest.mark.parametrize( "path, dest, is_file, expected_path", [ - ("src", None, False, Path("output") / "src.zip"), - ("src/", None, False, Path("output") / "src.zip"), - ("src", "source", False, Path("output") / "source" / "src.zip"), - ("src/app.py", None, True, Path("output") / "src" / "app.py"), + ("src", None, False, bundle_root / "src.zip"), + ("src/", None, False, bundle_root / "src.zip"), + ("src", "source", False, bundle_root / "source" / "src.zip"), + ("src/app.py", None, True, bundle_root / "src" / "app.py"), ( "src/app.py", "source/new_app.py", True, - Path("output") / "source" / "new_app.py", + bundle_root / "source" / "new_app.py", ), - ("src/*", "source/new_app.py", True, Path("output") / "source" / "new_app.py"), + ("src/*", "source/new_app.py", True, bundle_root / "source" / "new_app.py"), ( "src/dir/dir2/app.py", None, True, - Path("output") / "src" / "dir" / "dir2" / "app.py", + bundle_root / "src" / "dir" / "dir2" / "app.py", ), ( "src/dir/dir2/app.py", "source/", True, - Path("output") / "source" / "app.py", + bundle_root / "source" / "app.py", ), - ("src/*", "source/", False, Path("output") / "source" / "src.zip"), - ("src/**/*.py", None, False, Path("output") / "src.zip"), - ("src/**/*.py", "source/", False, Path("output") / "source" / "src.zip"), - ("src/app*", None, False, Path("output") / "src.zip"), - ("src/app[1-5].py", None, False, Path("output") / "src.zip"), + ("src/*", "source/", False, bundle_root / "source" / "src.zip"), + ("src/**/*.py", None, False, bundle_root / "src.zip"), + ("src/**/*.py", "source/", False, bundle_root / "source" / "src.zip"), + ("src/app*", None, False, bundle_root / "src.zip"), + ("src/app[1-5].py", None, False, bundle_root / "src.zip"), ], ) def test_artifact_post_build_path(path, dest, is_file, expected_path): with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - post_build_path = Artefact(Path(), Path(path), dest).post_build_path + post_build_path = Artefact( + Path(), bundle_root, Path(path), dest + ).post_build_path assert post_build_path == expected_path @@ -134,7 +141,12 @@ def test_artifact_import_path_from_other_directory( stage = "stage" with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - import_path = Artefact(Path("/tmp"), Path(path), dest).import_path(stage) + import_path = Artefact( + Path("/tmp"), + Path("/tmp") / "output" / "deploy" / "snowpark", + Path(path), + dest, + ).import_path(stage) assert import_path == expected_path @@ -166,7 +178,9 @@ def test_artifact_upload_path_from_other_directory( mock_ctx_context.return_value.connection = mock_connection with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): - upload_path = Artefact(Path("/tmp"), Path(path), dest).upload_path("stage") + upload_path = Artefact( + Path("/tmp"), Path("/tmp") / "output" / "deploy", Path(path), dest + ).upload_path("stage") assert upload_path == expected_path @@ -174,48 +188,48 @@ def test_artifact_upload_path_from_other_directory( @pytest.mark.parametrize( "path, dest, is_file, expected_path", [ - ("src", None, False, Path.cwd().absolute() / "output" / "src.zip"), - ("src/", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src", None, False, absolute_bundle_root / "src.zip"), + ("src/", None, False, absolute_bundle_root / "src.zip"), ( "src", "source", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_bundle_root / "source" / "src.zip", ), - ("src/app.py", None, True, Path.cwd().absolute() / "output" / "src" / "app.py"), + ("src/app.py", None, True, absolute_bundle_root / "src" / "app.py"), ( "src/app.py", "source/new_app.py", True, - Path.cwd().absolute() / "output" / "source" / "new_app.py", + absolute_bundle_root / "source" / "new_app.py", ), ( "src/dir/dir2/app.py", None, True, - Path.cwd().absolute() / "output" / "src" / "dir" / "dir2" / "app.py", + absolute_bundle_root / "src" / "dir" / "dir2" / "app.py", ), ( "src/dir/dir2/app.py", "source/", True, - Path.cwd().absolute() / "output" / "source" / "app.py", + absolute_bundle_root / "source" / "app.py", ), ( "src/*", "source/", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_bundle_root / "source" / "src.zip", ), - ("src/**/*.py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/**/*.py", None, False, absolute_bundle_root / "src.zip"), ( "src/**/*.py", "source/", False, - Path.cwd().absolute() / "output" / "source" / "src.zip", + absolute_bundle_root / "source" / "src.zip", ), - ("src/app*", None, False, Path.cwd().absolute() / "output" / "src.zip"), - ("src/app[1-5].py", None, False, Path.cwd().absolute() / "output" / "src.zip"), + ("src/app*", None, False, absolute_bundle_root / "src.zip"), + ("src/app[1-5].py", None, False, absolute_bundle_root / "src.zip"), ], ) def test_artifact_post_build_path_from_other_directory( @@ -223,7 +237,10 @@ def test_artifact_post_build_path_from_other_directory( ): with mock.patch.object(Path, "is_file" if is_file else "is_dir", return_value=True): post_build_path = Artefact( - Path.cwd().absolute(), Path(path), dest + Path.cwd().absolute(), + absolute_bundle_root, + Path(path), + dest, ).post_build_path assert post_build_path == expected_path diff --git a/tests/streamlit/test_artifacts.py b/tests/streamlit/test_artifacts.py index d63f3033df..fe05ffab9a 100644 --- a/tests/streamlit/test_artifacts.py +++ b/tests/streamlit/test_artifacts.py @@ -6,6 +6,8 @@ from snowflake.cli._plugins.connection.util import UIParameter from snowflake.connector.compat import IS_WINDOWS +bundle_root = Path("output") / "bundle" / "streamlit" + @pytest.mark.parametrize( "artifacts, paths", @@ -13,9 +15,9 @@ ( "src", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], @@ -23,9 +25,9 @@ ( "src/", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], @@ -33,19 +35,19 @@ ( "src/*", [ - {"local": Path("output") / "src" / "app.py", "stage": "/src"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", }, ], ), - ("src/*.py", [{"local": Path("output") / "src" / "app.py", "stage": "/src"}]), + ("src/*.py", [{"local": bundle_root / "src" / "app.py", "stage": "/src"}]), ( "src/dir/dir_app.py", [ { - "local": Path("output") / "src" / "dir" / "dir_app.py", + "local": bundle_root / "src" / "dir" / "dir_app.py", "stage": "/src/dir", } ], @@ -53,10 +55,10 @@ ( {"src": "src/**/*", "dest": "source/"}, [ - {"local": Path("output") / "source" / "app.py", "stage": "/source"}, - {"local": Path("output") / "source" / "dir_app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "dir_app.py", "stage": "/source"}, { - "local": Path("output") / "source" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "dir" / "dir_app.py", "stage": "/source/dir", }, ], @@ -65,11 +67,11 @@ {"src": "src", "dest": "source/"}, [ { - "local": Path("output") / "source" / "src" / "app.py", + "local": bundle_root / "source" / "src" / "app.py", "stage": "/source/src", }, { - "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -78,11 +80,11 @@ {"src": "src/", "dest": "source/"}, [ { - "local": Path("output") / "source" / "src" / "app.py", + "local": bundle_root / "source" / "src" / "app.py", "stage": "/source/src", }, { - "local": Path("output") / "source" / "src" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -90,9 +92,9 @@ ( {"src": "src/*", "dest": "source/"}, [ - {"local": Path("output") / "source" / "app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, { - "local": Path("output") / "source" / "dir" / "dir_app.py", + "local": bundle_root / "source" / "dir" / "dir_app.py", "stage": "/source/dir", }, ], @@ -101,7 +103,7 @@ {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, [ { - "local": Path("output") / "source" / "dir" / "apps" / "dir_app.py", + "local": bundle_root / "source" / "dir" / "apps" / "dir_app.py", "stage": "/source/dir/apps", } ], @@ -179,43 +181,63 @@ def test_deploy_with_artifacts( ( "src", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), ( "src/", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), ( "src/*", [ - {"local": "output/src/app.py", "stage": "/src"}, - {"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}, + {"local": bundle_root / "src" / "app.py", "stage": "/src"}, + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + }, ], ), - ("src/*.py", [{"local": "output/src/app.py", "stage": "/src"}]), + ("src/*.py", [{"local": bundle_root / "src" / "app.py", "stage": "/src"}]), ( "src/dir/dir_app.py", - [{"local": "output/src/dir/dir_app.py", "stage": "/src/dir"}], + [ + { + "local": bundle_root / "src" / "dir" / "dir_app.py", + "stage": "/src/dir", + } + ], ), ( {"src": "src/**/*", "dest": "source/"}, [ - {"local": "output/source/app.py", "stage": "/source"}, - {"local": "output/source/dir_app.py", "stage": "/source"}, - {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, + {"local": bundle_root / "source" / "dir_app.py", "stage": "/source"}, + { + "local": bundle_root / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, ], ), ( {"src": "src", "dest": "source/"}, [ - {"local": "output/source/src/app.py", "stage": "/source/src"}, { - "local": "output/source/src/dir/dir_app.py", + "local": bundle_root / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -223,9 +245,12 @@ def test_deploy_with_artifacts( ( {"src": "src/", "dest": "source/"}, [ - {"local": "output/source/src/app.py", "stage": "/source/src"}, { - "local": "output/source/src/dir/dir_app.py", + "local": bundle_root / "source" / "src" / "app.py", + "stage": "/source/src", + }, + { + "local": bundle_root / "source" / "src" / "dir" / "dir_app.py", "stage": "/source/src/dir", }, ], @@ -233,15 +258,18 @@ def test_deploy_with_artifacts( ( {"src": "src/*", "dest": "source/"}, [ - {"local": "output/source/app.py", "stage": "/source"}, - {"local": "output/source/dir/dir_app.py", "stage": "/source/dir"}, + {"local": bundle_root / "source" / "app.py", "stage": "/source"}, + { + "local": bundle_root / "source" / "dir" / "dir_app.py", + "stage": "/source/dir", + }, ], ), ( {"src": "src/dir/dir_app.py", "dest": "source/dir/apps/"}, [ { - "local": "output/source/dir/apps/dir_app.py", + "local": bundle_root / "source" / "dir" / "apps" / "dir_app.py", "stage": "/source/dir/apps", } ], diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index 9a859ad030..8eff0b4153 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -57,7 +57,7 @@ def test_describe_streamlit(mock_connector, runner, mock_ctx): def _put_query(project_root: Path, source: str, dest: str): return dedent( - f"put file://{project_root.resolve() / 'output' / source} {dest} auto_compress=false parallel=4 overwrite=True" + f"put file://{project_root.resolve() / 'output' / 'bundle' / 'streamlit' / source} {dest} auto_compress=false parallel=4 overwrite=True" ) diff --git a/tests_integration/test_snowpark.py b/tests_integration/test_snowpark.py index c55f33ee90..49cd4d5c96 100644 --- a/tests_integration/test_snowpark.py +++ b/tests_integration/test_snowpark.py @@ -33,6 +33,7 @@ STAGE_NAME = "dev_deployment" RETURN_TYPE = "VARCHAR" if IS_QA else "VARCHAR(16777216)" +bundle_root = Path("output") / "bundle" / "snowpark" @pytest.mark.integration @@ -48,8 +49,10 @@ def test_snowpark_flow( _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "my_snowpark_project", - Path("output") / "my_snowpark_project" / "app.zip", + Path("output") / "bundle", + bundle_root, + bundle_root / "my_snowpark_project", + bundle_root / "my_snowpark_project" / "app.zip", ] ) @@ -1248,9 +1251,11 @@ def test_snowpark_flow_v2( _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "app_1.zip", - Path("output") / "app_2.zip", - Path("output") / "c.py", + Path("output") / "bundle", + bundle_root, + bundle_root / "app_1.zip", + bundle_root / "app_2.zip", + bundle_root / "c.py", ] ) _test_steps.snowpark_deploy_should_finish_successfully_and_return( @@ -1395,9 +1400,11 @@ def test_snowpark_with_glob_patterns( _test_steps.snowpark_build_should_zip_files( additional_files=[ Path("output"), - Path("output") / "app_1.zip", - Path("output") / "app_2.zip", - Path("output") / "e.py", + Path("output") / "bundle", + bundle_root, + bundle_root / "app_1.zip", + bundle_root / "app_2.zip", + bundle_root / "e.py", ] ) _test_steps.snowpark_deploy_should_finish_successfully_and_return( diff --git a/tests_integration/testing_utils/snowpark_utils.py b/tests_integration/testing_utils/snowpark_utils.py index 33412f2d0f..3cb4510bc7 100644 --- a/tests_integration/testing_utils/snowpark_utils.py +++ b/tests_integration/testing_utils/snowpark_utils.py @@ -172,7 +172,9 @@ def snowpark_build_should_zip_files( if not no_dependencies: if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled(): - additional_files.append(Path("output") / "dependencies.zip") + additional_files.append( + Path("output") / "bundle" / "snowpark" / "dependencies.zip" + ) else: additional_files.append(Path("dependencies.zip")) From e14a06cb504e61a36982beb02a797a14bec590c1 Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Thu, 12 Dec 2024 18:55:36 +0100 Subject: [PATCH 08/10] Updated release-notes --- RELEASE-NOTES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 9c4e751adf..d2d3b9ee32 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -25,6 +25,8 @@ * `snow app release-directive unset` * Add support for release channels feature in native app version creation/drop. * `snow app version create` now returns version, patch, and label in JSON format. +* Added support for glob pattern in artifact paths in snowflake.yml for Streamlit. +* Added support for glob pattern in artifact paths in snowflake.yml for Snowpark, requires ENABLE_SNOWPARK_GLOB_SUPPORT feature flag. ## Fixes and improvements * Fixed crashes with older x86_64 Intel CPUs. @@ -51,8 +53,6 @@ * `--follow`: Stream logs in real-time. * `--follow-interval`: Set custom polling intervals during log streaming. * `snow connection add` supports `--no-interactive` flag to skip interactive prompts. -* Added support for glob pattern in artifact paths in snowflake.yml for Streamlit. -* Added support for glob pattern in artifact paths in snowflake.yml for Snowpark, requires ENABLE_SNOWPARK_GLOB_SUPPORT feature flag. ## Fixes and improvements * `snow --info` callback returns information about `SNOWFLAKE_HOME` variable. From 385b3aff5c7b93698d41ace927d6f0399da5ea7e Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Thu, 12 Dec 2024 18:58:30 +0100 Subject: [PATCH 09/10] Fixed import after rebase --- src/snowflake/cli/_plugins/streamlit/streamlit_entity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py b/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py index 6b187ba54b..8a364c5517 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py @@ -10,7 +10,7 @@ StreamlitEntityModel, ) from snowflake.cli.api.entities.common import EntityBase -from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping +from snowflake.cli.api.project.schemas.entities.common import PathMapping # WARNING: This entity is not implemented yet. The logic below is only for demonstrating the From c15cca502b782ff55c10d89f201fef5ff21bc1da Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Fri, 13 Dec 2024 11:06:56 +0100 Subject: [PATCH 10/10] Fixes after rebase --- src/snowflake/cli/_plugins/streamlit/streamlit_entity.py | 6 ++++-- tests/streamlit/test_streamlit_entity.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py b/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py index 8a364c5517..ce01bef368 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_entity.py @@ -33,7 +33,7 @@ def project_root(self) -> Path: @property def deploy_root(self) -> Path: - return self.project_root / "output" / "deploy" + return self.project_root / "output" / "bundle" / "streamlit" def action_bundle( self, @@ -47,7 +47,9 @@ def bundle(self, bundle_root=None): self.project_root, bundle_root or self.deploy_root, [ - PathMapping(src=str(artifact)) + PathMapping( + src=artifact.src, dest=artifact.dest, processors=artifact.processors + ) for artifact in self._entity_model.artifacts ], ) diff --git a/tests/streamlit/test_streamlit_entity.py b/tests/streamlit/test_streamlit_entity.py index 315e34b8e5..d1ba41ef2f 100644 --- a/tests/streamlit/test_streamlit_entity.py +++ b/tests/streamlit/test_streamlit_entity.py @@ -41,7 +41,7 @@ def test_nativeapp_children_interface(temp_dir): sl = StreamlitEntity(model, ctx) sl.bundle() - bundle_artifact = Path(temp_dir) / "output" / "deploy" / main_file + bundle_artifact = Path(temp_dir) / "output" / "bundle" / "streamlit" / main_file deploy_sql_str = sl.get_deploy_sql() grant_sql_str = sl.get_usage_grant_sql(app_role="app_role")