Added glob support for snowpark and streamlit (#1862)

* Added glob support for Snowpark and Streamlit * Added glob support for Snowpark and Streamlit * Added glob support for Snowpark and Streamlit * Changed PathMapping.src to Path * Changes after review * Changes after review 2 * Revert "Changed PathMapping.src to Path" This reverts commit 3eb543d. * Changes after review 3 * Changes after review 4 * Added full global support * Renamed feature flag and added release notes * Removed or None
snowflakedb · Nov 27, 2024 · 4b9022c · 4b9022c
1 parent 61ec0eb
commit 4b9022c
Show file tree

Hide file tree

Showing 58 changed files with 3,416 additions and 220 deletions.
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
@@ -42,6 +42,8 @@
   * `--follow`: Stream logs in real-time.
   * `--follow-interval`: Set custom polling intervals during log streaming.
 * `snow connection add` supports `--no-interactive` flag to skip interactive prompts.
+* Added support for glob pattern in artifact paths in snowflake.yml for Streamlit.
+* Added support for glob pattern in artifact paths in snowflake.yml for Snowpark, requires ENABLE_SNOWPARK_GLOB_SUPPORT feature flag.
 
 ## Fixes and improvements
 * `snow --info` callback returns information about `SNOWFLAKE_HOME` variable.

diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py
@@ -4,7 +4,7 @@
 import re
 from pathlib import Path
 from textwrap import dedent
-from typing import List, Literal, Optional, Union
+from typing import List, Literal, Optional
 
 import typer
 from click import BadOptionUsage, ClickException
@@ -64,6 +64,7 @@
 )
 from snowflake.cli.api.errno import DOES_NOT_EXIST_OR_NOT_AUTHORIZED
 from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError
+from snowflake.cli.api.project.schemas.commons import Artifacts
 from snowflake.cli.api.project.schemas.entities.common import (
     EntityModelBase,
     Identifier,
@@ -90,7 +91,7 @@
 
 class ApplicationPackageEntityModel(EntityModelBase):
     type: Literal["application package"] = DiscriminatorField()  # noqa: A003
-    artifacts: List[Union[PathMapping, str]] = Field(
+    artifacts: Artifacts = Field(
         title="List of paths or file source/destination pairs to add to the deploy root",
     )
     bundle_root: Optional[str] = Field(
@@ -137,10 +138,8 @@ def append_test_resource_suffix_to_identifier(
 
     @field_validator("artifacts")
     @classmethod
-    def transform_artifacts(
-        cls, orig_artifacts: List[Union[PathMapping, str]]
-    ) -> List[PathMapping]:
-        transformed_artifacts = []
+    def transform_artifacts(cls, orig_artifacts: Artifacts) -> List[PathMapping]:
+        transformed_artifacts: List[PathMapping] = []
         if orig_artifacts is None:
             return transformed_artifacts
 

diff --git a/src/snowflake/cli/_plugins/snowpark/commands.py b/src/snowflake/cli/_plugins/snowpark/commands.py
@@ -20,6 +20,7 @@
 
 import typer
 from click import ClickException, UsageError
+from snowflake.cli._plugins.nativeapp.artifacts import BundleMap, symlink_or_copy
 from snowflake.cli._plugins.object.commands import (
     describe as object_describe,
 )
@@ -59,7 +60,7 @@
     IndexUrlOption,
     SkipVersionCheckOption,
 )
-from snowflake.cli._plugins.snowpark.zipper import zip_dir
+from snowflake.cli._plugins.snowpark.zipper import zip_dir, zip_dir_using_bundle_map
 from snowflake.cli._plugins.stage.manager import StageManager
 from snowflake.cli.api.cli_global_context import (
     get_cli_context,
@@ -81,6 +82,7 @@
 from snowflake.cli.api.exceptions import (
     SecretsWithoutExternalAccessIntegrationError,
 )
+from snowflake.cli.api.feature_flags import FeatureFlag
 from snowflake.cli.api.identifiers import FQN
 from snowflake.cli.api.output.types import (
     CollectionResult,
@@ -95,6 +97,7 @@
     ProjectDefinition,
     ProjectDefinitionV2,
 )
+from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping
 from snowflake.cli.api.secure_path import SecurePath
 from snowflake.connector import DictCursor, ProgrammingError
 from snowflake.connector.cursor import SnowflakeCursor
@@ -225,8 +228,8 @@ def build_artifacts_mappings(
             entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage))
         stages_to_artifact_map[stage].update(required_artifacts)
 
-        if project_paths.dependencies.exists():
-            deps_artefact = project_paths.get_dependencies_artefact()
+        deps_artefact = project_paths.get_dependencies_artefact()
+        if deps_artefact.post_build_path.exists():
             stages_to_artifact_map[stage].add(deps_artefact)
             entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage))
     return entities_to_imports_map, stages_to_artifact_map
@@ -239,11 +242,12 @@ def create_stages_and_upload_artifacts(stages_to_artifact_map: StageToArtefactMa
         stage = FQN.from_stage(stage).using_context()
         stage_manager.create(fqn=stage, comment="deployments managed by Snowflake CLI")
         for artefact in artifacts:
+            post_build_path = artefact.post_build_path
             cli_console.step(
-                f"Uploading {artefact.post_build_path.name} to {artefact.upload_path(stage)}"
+                f"Uploading {post_build_path.name} to {artefact.upload_path(stage)}"
             )
             stage_manager.put(
-                local_path=artefact.post_build_path,
+                local_path=post_build_path,
                 stage_path=artefact.upload_path(stage),
                 overwrite=True,
             )
@@ -324,6 +328,9 @@ def build(
 
     anaconda_packages_manager = AnacondaPackagesManager()
 
+    # Clean up deploy root
+    project_paths.remove_up_deploy_root()
+
     # Resolve dependencies
     if project_paths.requirements.exists():
         with (
@@ -362,22 +369,50 @@ def build(
                 )
 
             if any(temp_deps_dir.path.iterdir()):
-                cli_console.step(f"Creating {project_paths.dependencies.name}")
+                dep_artifact = project_paths.get_dependencies_artefact()
+                cli_console.step(f"Creating {dep_artifact.path.name}")
                 zip_dir(
                     source=temp_deps_dir.path,
-                    dest_zip=project_paths.dependencies,
+                    dest_zip=dep_artifact.post_build_path,
                 )
             else:
                 cli_console.step(f"No external dependencies.")
 
     artifacts = set()
-    for entity in get_snowpark_entities(pd).values():
-        artifacts.update(entity.artifacts)
-
     with cli_console.phase("Preparing artifacts for source code"):
-        for artefact in artifacts:
-            artefact_dto = project_paths.get_artefact_dto(artefact)
-            artefact_dto.build()
+        if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
+            for entity in get_snowpark_entities(pd).values():
+                for artifact in entity.artifacts:
+                    artifacts.add(project_paths.get_artefact_dto(artifact))
+
+            for artefact in artifacts:
+                bundle_map = BundleMap(
+                    project_root=artefact.project_root,
+                    deploy_root=project_paths.deploy_root,
+                )
+                bundle_map.add(PathMapping(src=str(artefact.path), dest=artefact.dest))
+
+                if artefact.path.is_file():
+                    for (absolute_src, absolute_dest) in bundle_map.all_mappings(
+                        absolute=True, expand_directories=False
+                    ):
+                        symlink_or_copy(
+                            absolute_src,
+                            absolute_dest,
+                            deploy_root=bundle_map.deploy_root(),
+                        )
+                else:
+                    zip_dir_using_bundle_map(
+                        bundle_map=bundle_map,
+                        dest_zip=artefact.post_build_path,
+                    )
+        else:
+            for entity in get_snowpark_entities(pd).values():
+                for artifact in entity.artifacts:
+                    artifacts.add(project_paths.get_artefact_dto(artifact))
+
+            for artefact in artifacts:
+                artefact.build()
 
     return MessageResult(f"Build done.")
 

diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py b/src/snowflake/cli/_plugins/snowpark/snowpark_entity_model.py
@@ -14,36 +14,25 @@
 
 from __future__ import annotations
 
-from pathlib import Path
+import glob
 from typing import List, Literal, Optional, Union
 
 from pydantic import Field, field_validator
+from snowflake.cli.api.feature_flags import FeatureFlag
 from snowflake.cli.api.identifiers import FQN
+from snowflake.cli.api.project.schemas.commons import Artifacts
 from snowflake.cli.api.project.schemas.entities.common import (
     EntityModelBase,
     ExternalAccessBaseModel,
     ImportsBaseModel,
 )
 from snowflake.cli.api.project.schemas.updatable_model import (
     DiscriminatorField,
-    UpdatableModel,
 )
+from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping
 from snowflake.cli.api.project.schemas.v1.snowpark.argument import Argument
 
 
-class PathMapping(UpdatableModel):
-    class Config:
-        frozen = True
-
-    src: Path = Field(title="Source path (relative to project root)", default=None)
-
-    dest: Optional[str] = Field(
-        title="Destination path on stage",
-        description="Paths are relative to stage root; paths ending with a slash indicate that the destination is a directory which source files should be copied into.",
-        default=None,
-    )
-
-
 class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel):
     handler: str = Field(
         title="Function’s or procedure’s implementation of the object inside source module",
@@ -59,17 +48,24 @@ class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseM
         title="Python version to use when executing ", default=None
     )
     stage: str = Field(title="Stage in which artifacts will be stored")
-    artifacts: List[Union[PathMapping, str]] = Field(title="List of required sources")
+    artifacts: Artifacts = Field(title="List of required sources")
 
     @field_validator("artifacts")
     @classmethod
     def _convert_artifacts(cls, artifacts: Union[dict, str]):
         _artifacts = []
-        for artefact in artifacts:
-            if isinstance(artefact, PathMapping):
-                _artifacts.append(artefact)
+        for artifact in artifacts:
+            if (
+                (isinstance(artifact, str) and glob.has_magic(artifact))
+                or (isinstance(artifact, PathMapping) and glob.has_magic(artifact.src))
+            ) and FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_disabled():
+                raise ValueError(
+                    "If you want to use glob patterns in artifacts, you need to enable the Snowpark new build feature flag (ENABLE_SNOWPARK_GLOB_SUPPORT=true)"
+                )
+            if isinstance(artifact, PathMapping):
+                _artifacts.append(artifact)
             else:
-                _artifacts.append(PathMapping(src=artefact))
+                _artifacts.append(PathMapping(src=artifact))
         return _artifacts
 
     @field_validator("runtime")
@@ -79,14 +75,6 @@ def convert_runtime(cls, runtime_input: Union[str, float]) -> str:
             return str(runtime_input)
         return runtime_input
 
-    @field_validator("artifacts")
-    @classmethod
-    def validate_artifacts(cls, artifacts: List[Path]) -> List[Path]:
-        for artefact in artifacts:
-            if "*" in str(artefact):
-                raise ValueError("Glob patterns not supported for Snowpark artifacts.")
-        return artifacts
-
     @property
     def udf_sproc_identifier(self) -> UdfSprocIdentifier:
         return UdfSprocIdentifier.from_definition(self)