From 86172521826a809b8df308332760bad37b987af7 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Fri, 29 Nov 2024 14:59:05 -0500 Subject: [PATCH 01/33] teardown with comments --- .gitignore | 1 + .../codegen/templates/templates_processor.py | 67 ++++++++++--------- .../cli/_plugins/nativeapp/commands.py | 38 ++++++++++- .../cli/_plugins/nativeapp/constants.py | 3 + .../nativeapp/entities/application.py | 23 +++++-- .../nativeapp/entities/application_package.py | 45 +++++++++++-- .../cli/_plugins/nativeapp/sf_sql_facade.py | 24 +++++++ src/snowflake/cli/_plugins/stage/diff.py | 32 +++++++-- .../cli/_plugins/workspace/manager.py | 18 +++++ src/snowflake/cli/api/entities/utils.py | 19 ++++-- src/snowflake/cli/api/project/util.py | 18 +++++ 11 files changed, 230 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index cdc3b6490a..fbac1aaaf9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ gen_docs/ .env .vscode tmp/ +py11venv/ ^app.zip ^snowflake.yml diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index 9e38eecc2c..1f98a8dc4b 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -57,39 +57,44 @@ def expand_templates_in_file( """ if src.is_dir(): return - - with self.edit_file(dest) as file: - if not has_client_side_templates(file.contents) and not ( - _is_sql_file(dest) and has_sql_templates(file.contents) - ): - return - - src_file_name = src.relative_to(self._bundle_ctx.project_root) - cc.step(f"Expanding templates in {src_file_name}") - with cc.indented(): - try: - jinja_env = ( - choose_sql_jinja_env_based_on_template_syntax( - file.contents, reference_name=src_file_name + src_file_name = src.relative_to(self._bundle_ctx.project_root) + try: + with self.edit_file(dest) as file: + if not has_client_side_templates(file.contents) and not ( + _is_sql_file(dest) and has_sql_templates(file.contents) + ): + return + cc.step(f"Expanding templates in {src_file_name}") + with cc.indented(): + try: + jinja_env = ( + choose_sql_jinja_env_based_on_template_syntax( + file.contents, reference_name=src_file_name + ) + if _is_sql_file(dest) + else get_client_side_jinja_env() + ) + expanded_template = jinja_env.from_string(file.contents).render( + template_context or get_cli_context().template_context ) - if _is_sql_file(dest) - else get_client_side_jinja_env() - ) - expanded_template = jinja_env.from_string(file.contents).render( - template_context or get_cli_context().template_context - ) - - # For now, we are printing the source file path in the error message - # instead of the destination file path to make it easier for the user - # to identify the file that has the error, and edit the correct file. - except jinja2.TemplateSyntaxError as e: - raise InvalidTemplateInFileError(src_file_name, e, e.lineno) from e - - except jinja2.UndefinedError as e: - raise InvalidTemplateInFileError(src_file_name, e) from e - if expanded_template != file.contents: - file.edited_contents = expanded_template + # For now, we are printing the source file path in the error message + # instead of the destination file path to make it easier for the user + # to identify the file that has the error, and edit the correct file. + except jinja2.TemplateSyntaxError as e: + raise InvalidTemplateInFileError( + src_file_name, e, e.lineno + ) from e + + except jinja2.UndefinedError as e: + raise InvalidTemplateInFileError(src_file_name, e) from e + + if expanded_template != file.contents: + file.edited_contents = expanded_template + except UnicodeDecodeError as err: + cc.warning( + f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." + ) @span("templates_processor") def process( diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index b60ea157fa..c728e9051a 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -21,6 +21,7 @@ from textwrap import dedent from typing import Generator, Iterable, List, Optional, cast +import pydevd_pycharm import typer from snowflake.cli._plugins.nativeapp.common_flags import ( ForceOption, @@ -31,6 +32,7 @@ from snowflake.cli._plugins.nativeapp.entities.application_package import ( ApplicationPackageEntityModel, ) +from snowflake.cli._plugins.nativeapp.sf_facade import get_snowflake_facade from snowflake.cli._plugins.nativeapp.v2_conversions.compat import ( find_entity, force_project_definition_v2, @@ -61,6 +63,10 @@ ) from typing_extensions import Annotated +pydevd_pycharm.settrace( + "localhost", port=12345, stdoutToServer=True, stderrToServer=True, suspend=False +) + app = SnowTyperFactory( name="app", help="Manages a Snowflake Native App", @@ -115,7 +121,9 @@ def app_diff( ) stage_fqn = f"{package.fqn.name}.{package.stage}" diff: DiffResult = compute_stage_diff( - local_root=Path(package.deploy_root), stage_fqn=stage_fqn + local_root=Path(package.deploy_root) / Path(package.stage_subdirectory), + stage_fqn=stage_fqn, + stage_subdirectory=package.stage_subdirectory, ) if cli_context.output_format == OutputFormat.JSON: return ObjectResult(diff.to_dict()) @@ -246,11 +254,34 @@ def app_teardown( project_definition=cli_context.project_definition, project_root=cli_context.project_root, ) + + ## PJ-TODO for this PR maybe for dropping remote apps + # 0. create a quoted package with lower case (in snowsight or in yml) + # 1. make sure logic can get the applications and filter them right + # 2. create apps with quotes and lowercase and uppercase or without quotes and try the comparison between yml and remote + # 3. write a robust diff logic based on above + + # 0. encapsulate the application drop stuff so we can use it outside an app entity + + ## GET ALL APPS FILTERED + # app_names = get_snowflake_facade().get_all_applications_for_package( + # app_package_entity.identifier + # ) + + # PJ-TODO: add messaging here for extra packages found + all_packages_with_id = [ + package_entity.entity_id + for package_entity in project.get_entities_by_type( + ApplicationPackageEntityModel.get_type() + ).values() + if package_entity.identifier == app_package_entity.identifier + ] + + # PJ-TODO: fix messaging for all apps about to be dropped. we only show the one's that we could get with get_existing_app_info for app_entity in project.get_entities_by_type( ApplicationEntityModel.get_type() ).values(): - # Drop each app - if app_entity.from_.target == app_package_entity.entity_id: + if app_entity.from_.target in all_packages_with_id: ws.perform_action( app_entity.entity_id, EntityActions.DROP, @@ -258,6 +289,7 @@ def app_teardown( interactive=interactive, cascade=cascade, ) + # Then drop the package ws.perform_action( app_package_entity.entity_id, diff --git a/src/snowflake/cli/_plugins/nativeapp/constants.py b/src/snowflake/cli/_plugins/nativeapp/constants.py index 11d439b37f..fd9e3934f3 100644 --- a/src/snowflake/cli/_plugins/nativeapp/constants.py +++ b/src/snowflake/cli/_plugins/nativeapp/constants.py @@ -16,12 +16,15 @@ SPECIAL_COMMENT = "GENERATED_BY_SNOWFLAKECLI" ALLOWED_SPECIAL_COMMENTS = {SPECIAL_COMMENT, SPECIAL_COMMENT_OLD} LOOSE_FILES_MAGIC_VERSION = "UNVERSIONED" +APPLICATION_PACKAGE = "APPLICATION PACKAGE" NAME_COL = "name" COMMENT_COL = "comment" OWNER_COL = "owner" VERSION_COL = "version" PATCH_COL = "patch" +SOURCE_COL = "source" +SOURCE_TYPE_COL = "source_type" AUTHORIZE_TELEMETRY_COL = "authorize_telemetry_event_sharing" INTERNAL_DISTRIBUTION = "internal" diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application.py b/src/snowflake/cli/_plugins/nativeapp/entities/application.py index 040e60e82f..8bb0cb3c4e 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application.py @@ -30,6 +30,7 @@ COMMENT_COL, NAME_COL, OWNER_COL, + SOURCE_COL, SPECIAL_COMMENT, ) from snowflake.cli._plugins.nativeapp.entities.application_package import ( @@ -53,6 +54,7 @@ SameAccountInstallMethod, ) from snowflake.cli._plugins.nativeapp.sf_facade import get_snowflake_facade +from snowflake.cli._plugins.nativeapp.sf_sql_facade import _same_identifier from snowflake.cli._plugins.nativeapp.utils import needs_confirmation from snowflake.cli._plugins.workspace.context import ActionContext from snowflake.cli.api.cli_global_context import get_cli_context, span @@ -368,7 +370,6 @@ def action_deploy( recursive: bool, paths: List[Path], validate: bool = ValidateOption, - stage_fqn: Optional[str] = None, interactive: bool = InteractiveOption, version: Optional[str] = None, patch: Optional[int] = None, @@ -383,7 +384,7 @@ def action_deploy( package_entity: ApplicationPackageEntity = action_ctx.get_entity( self.package_entity_id ) - stage_fqn = stage_fqn or package_entity.stage_fqn + stage_fqn = package_entity.stage_fqn if force: policy = AllowAlwaysPolicy() @@ -432,7 +433,6 @@ def action_deploy( recursive=True, paths=[], validate=validate, - stage_fqn=stage_fqn, interactive=interactive, force=force, ) @@ -646,10 +646,14 @@ def create_or_upgrade_app( model = self._entity_model console = self._workspace_ctx.console debug_mode = model.debug - + # PJ-TODO: in the future, the stage_fqn input to here could already have the subdir? stage_fqn = stage_fqn or package.stage_fqn stage_schema = extract_schema(stage_fqn) - + path_to_artifacts = ( + f"{stage_fqn}/{package.stage_subdirectory}" + if package.stage_subdirectory + else stage_fqn + ) sql_executor = get_sql_executor() with sql_executor.use_role(self.role): event_sharing = EventSharingHandler( @@ -672,13 +676,18 @@ def create_or_upgrade_app( app_role=self.role, show_app_row=show_app_row, ) + if not _same_identifier(show_app_row[SOURCE_COL], package.name): + # PJ- TODO: change to: are you sure you want to proceed? + raise ClickException( + "This application was not originally created from this package" + ) # If all the above checks are in order, proceed to upgrade try: console.step( f"Upgrading existing application object {self.name}." ) - using_clause = install_method.using_clause(stage_fqn) + using_clause = install_method.using_clause(path_to_artifacts) upgrade_cursor = sql_executor.execute_query( f"alter application {self.name} upgrade {using_clause}", ) @@ -775,7 +784,7 @@ def create_or_upgrade_app( ) authorize_telemetry_clause = f" AUTHORIZE_TELEMETRY_EVENT_SHARING = {str(new_authorize_event_sharing_value).upper()}" - using_clause = install_method.using_clause(stage_fqn) + using_clause = install_method.using_clause(path_to_artifacts) create_cursor = sql_executor.execute_query( dedent( f"""\ diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 83f32d8b52..bc4f7c242f 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -121,6 +121,10 @@ class ApplicationPackageEntityModel(EntityModelBase): title="Path to manifest.yml. Unused and deprecated starting with Snowflake CLI 3.2", default="", ) + stage_subdirectory: Optional[str] = Field( + title="Subfolder in stage", + default="", + ) @field_validator("identifier") @classmethod @@ -162,6 +166,10 @@ def validate_source_stage(cls, input_value: str): return input_value +class SyncDataClass: + pass + + @attach_spans_to_entity_actions(entity_name="app_pkg") class ApplicationPackageEntity(EntityBase[ApplicationPackageEntityModel]): """ @@ -172,9 +180,17 @@ class ApplicationPackageEntity(EntityBase[ApplicationPackageEntityModel]): def project_root(self) -> Path: return self._workspace_ctx.project_root + @property + def stage_subdirectory(self) -> str: + return self._entity_model.stage_subdirectory + @property def deploy_root(self) -> Path: - return self.project_root / self._entity_model.deploy_root + return ( + self.project_root + / self._entity_model.deploy_root + / self._entity_model.stage_subdirectory + ) @property def bundle_root(self) -> Path: @@ -204,6 +220,10 @@ def warehouse(self) -> str: def stage_fqn(self) -> str: return f"{self.name}.{self._entity_model.stage}" + # @property + # def stage_root_path(self) -> str: + # return f"{self.stage_fqn}/{self.stage_subdirectory}" if self.stage_subdirectory else self.stage_fqn + @property def scratch_stage_fqn(self) -> str: return f"{self.name}.{self._entity_model.scratch_stage}" @@ -225,7 +245,6 @@ def action_deploy( validate: bool, interactive: bool, force: bool, - stage_fqn: Optional[str] = None, *args, **kwargs, ): @@ -236,11 +255,15 @@ def action_deploy( paths=paths, print_diff=True, validate=validate, - stage_fqn=stage_fqn or self.stage_fqn, + stage_fqn=self.stage_fqn, interactive=interactive, force=force, ) + def action_teardown(self, action_ctx: ActionContext, *args, **kwargs): + + get_snowflake_facade().get_all_applications_for_package(self.name) + def action_drop(self, action_ctx: ActionContext, force_drop: bool, *args, **kwargs): console = self._workspace_ctx.console sql_executor = get_sql_executor() @@ -596,7 +619,10 @@ def _deploy( with get_sql_executor().use_role(self.role): # 3. Upload files from deploy root local folder to the above stage + # PJ-TODO: move extract_schema in sync stage_schema = extract_schema(stage_fqn) + # PJ-TODO: refactor stage into some object so we don't have to pass parts of it around. schema, subdir, fqn blah blah. like bundlemap. + # There might already be something like that for it diff = sync_deploy_root_with_stage( console=console, deploy_root=self.deploy_root, @@ -607,6 +633,7 @@ def _deploy( prune=prune, recursive=recursive, stage_fqn=stage_fqn, + stage_subdirectory=self.stage_subdirectory, local_paths_to_sync=paths, print_diff=print_diff, ) @@ -926,11 +953,19 @@ def get_validation_result( force=force, run_post_deploy_hooks=False, ) - prefixed_stage_fqn = StageManager.get_standard_stage_prefix(stage_fqn) + # prefixed_stage_fqn = StageManager.get_standard_stage_prefix(stage_fqn) + stage_fqn_with_subdir = ( + f"{stage_fqn}/{self.stage_subdirectory}" + if self.stage_subdirectory + else stage_fqn + ) + prefixed_stage_fqn_full = StageManager.get_standard_stage_prefix( + stage_fqn_with_subdir + ) sql_executor = get_sql_executor() try: cursor = sql_executor.execute_query( - f"call system$validate_native_app_setup('{prefixed_stage_fqn}')" + f"call system$validate_native_app_setup('{prefixed_stage_fqn_full}')" ) except ProgrammingError as err: if err.errno == DOES_NOT_EXIST_OR_NOT_AUTHORIZED: diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index b8cd77dbee..0bdca44691 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -18,6 +18,12 @@ from textwrap import dedent from typing import Any, Dict, List +from snowflake.cli._plugins.nativeapp.constants import ( + APPLICATION_PACKAGE, + NAME_COL, + SOURCE_COL, + SOURCE_TYPE_COL, +) from snowflake.cli._plugins.nativeapp.sf_facade_constants import UseObjectType from snowflake.cli._plugins.nativeapp.sf_facade_exceptions import ( CouldNotUseObjectError, @@ -503,6 +509,24 @@ def show_release_directives( ) return cursor.fetchall() + def get_all_applications_for_package(self, package_name, role: str | None = None): + # TODO: break lines + show_apps_query = "SHOW APPLICATIONS" + select_column_from_result_clause = f"SELECT {to_quoted_identifier(NAME_COL)} FROM table(result_scan(last_query_id()))" + filter_type_package_clause = f"{to_quoted_identifier(SOURCE_TYPE_COL)} = {to_string_literal(APPLICATION_PACKAGE)}" + filter_package_name_clause = f"{to_quoted_identifier(SOURCE_COL)} = {to_string_literal(package_name.upper())}" + filter_applications_query = f"{select_column_from_result_clause} where {filter_type_package_clause} and {filter_package_name_clause}" + + with self._use_role_optional(role): + self._sql_executor.execute_query(show_apps_query) + app_names = [ + app_row[0] + for app_row in self._sql_executor.execute_query( + filter_applications_query + ).fetchall() + ] + return app_names + # TODO move this to src/snowflake/cli/api/project/util.py in a separate # PR since it's codeowned by the CLI team diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index ab3b2e0d91..3fd6af1bcb 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -83,18 +83,30 @@ def enumerate_files(path: Path) -> List[Path]: return paths -def strip_stage_name(path: str) -> StagePathType: - """Returns the given stage path without the stage name as the first part.""" - return StagePathType(*path.split("/")[1:]) +def relative_to_stage_subdir(path: str, subdir: str | None = None) -> StagePathType: + path_parts = path.split("/") + # Remove stage name + path_parts.pop(0) + path_wo_stage_name = StagePathType(*path_parts) + if subdir: + # Find file path relative to stage subdirectory + subdir_path = StagePathType(subdir) + return path_wo_stage_name.relative_to(subdir_path) -def build_md5_map(list_stage_cursor: DictCursor) -> Dict[StagePathType, Optional[str]]: + return path_wo_stage_name + + +def build_md5_map( + list_stage_cursor: DictCursor, stage_subdir: str | None = None +) -> Dict[StagePathType, Optional[str]]: """ Returns a mapping of relative stage paths to their md5sums. """ + all_files = list_stage_cursor.fetchall() return { - strip_stage_name(file["name"]): file["md5"] - for file in list_stage_cursor.fetchall() + relative_to_stage_subdir(file["name"], stage_subdir): file["md5"] + for file in all_files } @@ -118,13 +130,19 @@ def preserve_from_diff( def compute_stage_diff( local_root: Path, stage_fqn: str, + stage_subdirectory: str | None = None, ) -> DiffResult: """ Diffs the files in a stage with a local folder. """ + stage_fqn_with_subdir = ( + f"{stage_fqn}/{stage_subdirectory}" if stage_subdirectory else stage_fqn + ) stage_manager = StageManager() local_files = enumerate_files(local_root) - remote_md5 = build_md5_map(stage_manager.list_files(stage_fqn)) + remote_files = stage_manager.list_files(stage_fqn_with_subdir) + + remote_md5 = build_md5_map(remote_files, stage_subdirectory) result: DiffResult = DiffResult() diff --git a/src/snowflake/cli/_plugins/workspace/manager.py b/src/snowflake/cli/_plugins/workspace/manager.py index 25b56d542f..75e47604b9 100644 --- a/src/snowflake/cli/_plugins/workspace/manager.py +++ b/src/snowflake/cli/_plugins/workspace/manager.py @@ -31,6 +31,7 @@ def __init__(self, project_definition: ProjectDefinition, project_root: Path): self._entities_cache: Dict[str, Entity] = {} self._project_definition: DefinitionV20 = project_definition self._project_root = project_root + self._entities_identifier_cache: Dict[str, list[Entity]] = {} def get_entity(self, entity_id: str): """ @@ -52,6 +53,23 @@ def get_entity(self, entity_id: str): self._entities_cache[entity_id] = entity_cls(entity_model, workspace_ctx) return self._entities_cache[entity_id] + # def get_entities_with_identifier(self, identifier: str): + # if identifier in self._entities_identifier_cache: + # return self._entities_cache[entity_id] + # entity_model = self._project_definition.entities.get(entity_id, None) + # if entity_model is None: + # raise ValueError(f"No such entity ID: {entity_id}") + # entity_model_cls = entity_model.__class__ + # entity_cls = v2_entity_model_to_entity_map[entity_model_cls] + # workspace_ctx = WorkspaceContext( + # console=cc, + # project_root=self.project_root, + # get_default_role=_get_default_role, + # get_default_warehouse=_get_default_warehouse, + # ) + # self._entities_cache[entity_id] = entity_cls(entity_model, workspace_ctx) + # return self._entities_cache[entity_id] + def perform_action(self, entity_id: str, action: EntityActions, *args, **kwargs): """ Instantiates an entity of the given ID and calls the given action on it. diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 4c5b8b0c78..46af1de0a7 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -87,6 +87,7 @@ def sync_deploy_root_with_stage( prune: bool, recursive: bool, stage_fqn: str, + stage_subdirectory: str | None = None, local_paths_to_sync: List[Path] | None = None, print_diff: bool = True, ) -> DiffResult: @@ -107,7 +108,6 @@ def sync_deploy_root_with_stage( Returns: A `DiffResult` instance describing the changes that were performed. """ - sql_facade = get_snowflake_facade() # Does a stage already exist within the application package, or we need to create one? # Using "if not exists" should take care of either case. @@ -119,12 +119,21 @@ def sync_deploy_root_with_stage( sql_facade.create_stage(stage_fqn) # Perform a diff operation and display results to the user for informational purposes + # PJ - TODO: make optional / + # PJ - rename this + stage_fqn_with_subdir = ( + f"{stage_fqn}/{stage_subdirectory}" if stage_subdirectory else stage_fqn + ) if print_diff: console.step( - "Performing a diff between the Snowflake stage and your local deploy_root ('%s') directory." - % deploy_root.resolve() + f"Performing a diff between the Snowflake stage {stage_fqn_with_subdir} and your local deploy_root {deploy_root.resolve()} directory." ) - diff: DiffResult = compute_stage_diff(deploy_root, stage_fqn) + + diff: DiffResult = compute_stage_diff( + local_root=deploy_root, + stage_fqn=stage_fqn, + stage_subdirectory=stage_subdirectory, + ) if local_paths_to_sync: # Deploying specific files/directories @@ -185,7 +194,7 @@ def sync_deploy_root_with_stage( role=role, deploy_root_path=deploy_root, diff_result=diff, - stage_fqn=stage_fqn, + stage_fqn=stage_fqn_with_subdir, ) return diff diff --git a/src/snowflake/cli/api/project/util.py b/src/snowflake/cli/api/project/util.py index 564ebbd867..dcb1c50b8f 100644 --- a/src/snowflake/cli/api/project/util.py +++ b/src/snowflake/cli/api/project/util.py @@ -194,6 +194,24 @@ def extract_schema(qualified_name: str): return None +# def extract_schema_from_path_to_version_directory(path: str) -> str: +# """ +# Extracts the schema from a path to a version directory. +# Path to version directory is a stage fqn with an optional subdirectory. It can take any of these forms: +# db.schema.stage +# db.schema.stage/subdirectory +# schema.stage +# schema.stage/subdirectory +# """ +# DB_SCHEMA_AND_NAME = r"(?P[^.]+)\.(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" +# SCHEMA_AND_NAME = r"(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" +# +# match = re.match(DB_SCHEMA_AND_NAME, path) or re.match(SCHEMA_AND_NAME, path) +# if match: +# return match.groupdict()["schema"] +# return None + + def first_set_env(*keys: str): for k in keys: v = os.getenv(k) From 5a3e154fedbaa88e858437f099dc738554d0ccdb Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Fri, 29 Nov 2024 15:57:34 -0500 Subject: [PATCH 02/33] make version work --- src/snowflake/cli/_plugins/nativeapp/commands.py | 1 - .../nativeapp/entities/application_package.py | 5 +++-- .../cli/_plugins/nativeapp/sf_sql_facade.py | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index c728e9051a..2fb04cfc3b 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -32,7 +32,6 @@ from snowflake.cli._plugins.nativeapp.entities.application_package import ( ApplicationPackageEntityModel, ) -from snowflake.cli._plugins.nativeapp.sf_facade import get_snowflake_facade from snowflake.cli._plugins.nativeapp.v2_conversions.compat import ( find_entity, force_project_definition_v2, diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index bc4f7c242f..8ade2e0c87 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -437,6 +437,7 @@ def action_version_create( self.get_existing_release_directive_info_for_version(resolved_version) ) except InsufficientPrivilegesError: + # PJ - TODO: insufficient privileges error will still show the "are you sure you want to create a new patch? prompt. it shouldn't" warning = ( "Could not check for existing release directives due to insufficient privileges. " "The MANAGE RELEASES privilege is required to check for existing release directives." @@ -711,7 +712,7 @@ def add_new_version(self, version: str, label: str | None = None) -> None: get_snowflake_facade().create_version_in_package( role=self.role, package_name=self.name, - stage_fqn=self.stage_fqn, + stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" if self.stage_subdirectory else self.stage_fqn, version=version, label=label, ) @@ -735,7 +736,7 @@ def add_new_patch_to_version( new_patch = get_snowflake_facade().add_patch_to_package_version( role=self.role, package_name=self.name, - stage_fqn=self.stage_fqn, + stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" if self.stage_subdirectory else self.stage_fqn, version=version, patch=patch, label=label, diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index 0bdca44691..b2bc158657 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -198,7 +198,7 @@ def get_account_event_table(self, role: str | None = None) -> str | None: def create_version_in_package( self, package_name: str, - stage_fqn: str, + stage_fqn_with_subdir: str, version: str, label: str | None = None, role: str | None = None, @@ -206,7 +206,7 @@ def create_version_in_package( """ Creates a new version in an existing application package. @param package_name: Name of the application package to alter. - @param stage_fqn: Stage fully qualified name. + @param stage_fqn_with_subdir: Stage fully qualified name. @param version: Version name to create. @param [Optional] role: Switch to this role while executing create version. @param [Optional] label: Label for this version, visible to consumers. @@ -223,7 +223,7 @@ def create_version_in_package( f"""\ alter application package {package_name} add version {version} - using @{stage_fqn}{with_label_cause} + using @{stage_fqn_with_subdir}{with_label_cause} """ ) with self._use_role_optional(role): @@ -235,10 +235,11 @@ def create_version_in_package( f"Failed to add version {version} to application package {package_name}.", ) + # PJ-TODO: rename stage_fqn_with_subdir everywhere to a better name like stage_path_to_artifacts def add_patch_to_package_version( self, package_name: str, - stage_fqn: str, + stage_fqn_with_subdir: str, version: str, patch: int | None = None, label: str | None = None, @@ -247,7 +248,7 @@ def add_patch_to_package_version( """ Add a new patch, optionally a custom one, to an existing version in an application package. @param package_name: Name of the application package to alter. - @param stage_fqn: Stage fully qualified name. + @param stage_fqn_with_subdir: Stage fully qualified name. @param version: Version name to create. @param [Optional] patch: Patch number to create. @param [Optional] label: Label for this patch, visible to consumers. @@ -268,7 +269,7 @@ def add_patch_to_package_version( f"""\ alter application package {package_name} add patch {patch_query} for version {version} - using @{stage_fqn}{with_label_clause} + using @{stage_fqn_with_subdir}{with_label_clause} """ ) with self._use_role_optional(role): From 8db7a4040e7fe45b5631521989fa0f59ec536958 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Mon, 2 Dec 2024 09:32:47 -0500 Subject: [PATCH 03/33] update path for version create --- .../_plugins/nativeapp/entities/application_package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 8ade2e0c87..49cb203067 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -712,7 +712,9 @@ def add_new_version(self, version: str, label: str | None = None) -> None: get_snowflake_facade().create_version_in_package( role=self.role, package_name=self.name, - stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" if self.stage_subdirectory else self.stage_fqn, + stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" + if self.stage_subdirectory + else self.stage_fqn, version=version, label=label, ) @@ -736,7 +738,9 @@ def add_new_patch_to_version( new_patch = get_snowflake_facade().add_patch_to_package_version( role=self.role, package_name=self.name, - stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" if self.stage_subdirectory else self.stage_fqn, + stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" + if self.stage_subdirectory + else self.stage_fqn, version=version, patch=patch, label=label, From d9eb5f4fe5d881cdeb890f03cd3dbdc7cc398e2e Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Mon, 2 Dec 2024 09:39:01 -0500 Subject: [PATCH 04/33] catch tempalte processor file read error --- .../codegen/templates/templates_processor.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index 1f98a8dc4b..006afe2423 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -57,9 +57,17 @@ def expand_templates_in_file( """ if src.is_dir(): return + src_file_name = src.relative_to(self._bundle_ctx.project_root) + try: - with self.edit_file(dest) as file: + file = self.edit_file(dest) + except UnicodeDecodeError as err: + cc.warning( + f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." + ) + else: + with file: if not has_client_side_templates(file.contents) and not ( _is_sql_file(dest) and has_sql_templates(file.contents) ): @@ -91,10 +99,6 @@ def expand_templates_in_file( if expanded_template != file.contents: file.edited_contents = expanded_template - except UnicodeDecodeError as err: - cc.warning( - f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." - ) @span("templates_processor") def process( From 71108d01f22b47e67a8d4f50b2add3950bc4e56f Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Mon, 2 Dec 2024 11:22:06 -0500 Subject: [PATCH 05/33] make diff an entity action --- .../codegen/templates/templates_processor.py | 12 +++++----- .../cli/_plugins/nativeapp/commands.py | 22 +++++-------------- .../nativeapp/entities/application_package.py | 18 ++++++++++++++- src/snowflake/cli/api/entities/common.py | 1 + 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index 006afe2423..a16e1dc933 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -61,13 +61,7 @@ def expand_templates_in_file( src_file_name = src.relative_to(self._bundle_ctx.project_root) try: - file = self.edit_file(dest) - except UnicodeDecodeError as err: - cc.warning( - f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." - ) - else: - with file: + with self.edit_file(dest) as file: if not has_client_side_templates(file.contents) and not ( _is_sql_file(dest) and has_sql_templates(file.contents) ): @@ -99,6 +93,10 @@ def expand_templates_in_file( if expanded_template != file.contents: file.edited_contents = expanded_template + except UnicodeDecodeError as err: + cc.warning( + f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." + ) @span("templates_processor") def process( diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index 2fb04cfc3b..d55327889c 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -37,11 +37,6 @@ force_project_definition_v2, ) from snowflake.cli._plugins.nativeapp.version.commands import app as versions_app -from snowflake.cli._plugins.stage.diff import ( - DiffResult, - compute_stage_diff, -) -from snowflake.cli._plugins.stage.utils import print_diff_to_console from snowflake.cli._plugins.workspace.manager import WorkspaceManager from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.commands.decorators import ( @@ -113,22 +108,15 @@ def app_diff( project_root=cli_context.project_root, ) package_id = options["package_entity_id"] - package = cli_context.project_definition.entities[package_id] - bundle_map = ws.perform_action( + diff = ws.perform_action( package_id, - EntityActions.BUNDLE, - ) - stage_fqn = f"{package.fqn.name}.{package.stage}" - diff: DiffResult = compute_stage_diff( - local_root=Path(package.deploy_root) / Path(package.stage_subdirectory), - stage_fqn=stage_fqn, - stage_subdirectory=package.stage_subdirectory, + EntityActions.DIFF, + print_to_console=cli_context.output_format != OutputFormat.JSON, ) if cli_context.output_format == OutputFormat.JSON: return ObjectResult(diff.to_dict()) - else: - print_diff_to_console(diff, bundle_map) - return None # don't print any output + + return None @app.command("run", requires_connection=True) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 49cb203067..055a21eeac 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -46,8 +46,9 @@ InsufficientPrivilegesError, ) from snowflake.cli._plugins.nativeapp.utils import needs_confirmation -from snowflake.cli._plugins.stage.diff import DiffResult +from snowflake.cli._plugins.stage.diff import DiffResult, compute_stage_diff from snowflake.cli._plugins.stage.manager import StageManager +from snowflake.cli._plugins.stage.utils import print_diff_to_console from snowflake.cli._plugins.workspace.context import ActionContext from snowflake.cli.api.cli_global_context import span from snowflake.cli.api.entities.common import ( @@ -236,6 +237,21 @@ def post_deploy_hooks(self) -> list[PostDeployHook] | None: def action_bundle(self, action_ctx: ActionContext, *args, **kwargs): return self._bundle() + def action_diff( + self, action_ctx: ActionContext, print_to_console: bool, *args, **kwargs + ): + bundle_map = self._bundle() + diff = compute_stage_diff( + local_root=self.deploy_root, + stage_fqn=self.stage_fqn, + stage_subdirectory=self.stage_subdirectory, + ) + + if print_to_console: + print_diff_to_console(diff, bundle_map) + + return diff + def action_deploy( self, action_ctx: ActionContext, diff --git a/src/snowflake/cli/api/entities/common.py b/src/snowflake/cli/api/entities/common.py index 021830fcb2..f50968651c 100644 --- a/src/snowflake/cli/api/entities/common.py +++ b/src/snowflake/cli/api/entities/common.py @@ -12,6 +12,7 @@ class EntityActions(str, Enum): DROP = "action_drop" VALIDATE = "action_validate" EVENTS = "action_events" + DIFF = "action_diff" VERSION_LIST = "action_version_list" VERSION_CREATE = "action_version_create" From a7e3a20bf26082570db8488ad77b9132cce08598 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Mon, 2 Dec 2024 13:33:28 -0500 Subject: [PATCH 06/33] remove teardown with all apps logic --- .../cli/_plugins/nativeapp/commands.py | 18 +++----------- .../cli/_plugins/nativeapp/constants.py | 3 --- .../nativeapp/entities/application_package.py | 4 ---- .../cli/_plugins/nativeapp/sf_sql_facade.py | 24 ------------------- 4 files changed, 3 insertions(+), 46 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index d55327889c..a1fba60e42 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -242,20 +242,8 @@ def app_teardown( project_root=cli_context.project_root, ) - ## PJ-TODO for this PR maybe for dropping remote apps - # 0. create a quoted package with lower case (in snowsight or in yml) - # 1. make sure logic can get the applications and filter them right - # 2. create apps with quotes and lowercase and uppercase or without quotes and try the comparison between yml and remote - # 3. write a robust diff logic based on above - - # 0. encapsulate the application drop stuff so we can use it outside an app entity - - ## GET ALL APPS FILTERED - # app_names = get_snowflake_facade().get_all_applications_for_package( - # app_package_entity.identifier - # ) - - # PJ-TODO: add messaging here for extra packages found + # TODO: get all apps created from this application package from snowflake, compare, confirm and drop. + # TODO: add messaging/confirmation here for extra apps found as part of above all_packages_with_id = [ package_entity.entity_id for package_entity in project.get_entities_by_type( @@ -264,7 +252,7 @@ def app_teardown( if package_entity.identifier == app_package_entity.identifier ] - # PJ-TODO: fix messaging for all apps about to be dropped. we only show the one's that we could get with get_existing_app_info + for app_entity in project.get_entities_by_type( ApplicationEntityModel.get_type() ).values(): diff --git a/src/snowflake/cli/_plugins/nativeapp/constants.py b/src/snowflake/cli/_plugins/nativeapp/constants.py index fd9e3934f3..11d439b37f 100644 --- a/src/snowflake/cli/_plugins/nativeapp/constants.py +++ b/src/snowflake/cli/_plugins/nativeapp/constants.py @@ -16,15 +16,12 @@ SPECIAL_COMMENT = "GENERATED_BY_SNOWFLAKECLI" ALLOWED_SPECIAL_COMMENTS = {SPECIAL_COMMENT, SPECIAL_COMMENT_OLD} LOOSE_FILES_MAGIC_VERSION = "UNVERSIONED" -APPLICATION_PACKAGE = "APPLICATION PACKAGE" NAME_COL = "name" COMMENT_COL = "comment" OWNER_COL = "owner" VERSION_COL = "version" PATCH_COL = "patch" -SOURCE_COL = "source" -SOURCE_TYPE_COL = "source_type" AUTHORIZE_TELEMETRY_COL = "authorize_telemetry_event_sharing" INTERNAL_DISTRIBUTION = "internal" diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 055a21eeac..5ec557850e 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -276,10 +276,6 @@ def action_deploy( force=force, ) - def action_teardown(self, action_ctx: ActionContext, *args, **kwargs): - - get_snowflake_facade().get_all_applications_for_package(self.name) - def action_drop(self, action_ctx: ActionContext, force_drop: bool, *args, **kwargs): console = self._workspace_ctx.console sql_executor = get_sql_executor() diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index b2bc158657..be505bc7a1 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -18,12 +18,6 @@ from textwrap import dedent from typing import Any, Dict, List -from snowflake.cli._plugins.nativeapp.constants import ( - APPLICATION_PACKAGE, - NAME_COL, - SOURCE_COL, - SOURCE_TYPE_COL, -) from snowflake.cli._plugins.nativeapp.sf_facade_constants import UseObjectType from snowflake.cli._plugins.nativeapp.sf_facade_exceptions import ( CouldNotUseObjectError, @@ -510,24 +504,6 @@ def show_release_directives( ) return cursor.fetchall() - def get_all_applications_for_package(self, package_name, role: str | None = None): - # TODO: break lines - show_apps_query = "SHOW APPLICATIONS" - select_column_from_result_clause = f"SELECT {to_quoted_identifier(NAME_COL)} FROM table(result_scan(last_query_id()))" - filter_type_package_clause = f"{to_quoted_identifier(SOURCE_TYPE_COL)} = {to_string_literal(APPLICATION_PACKAGE)}" - filter_package_name_clause = f"{to_quoted_identifier(SOURCE_COL)} = {to_string_literal(package_name.upper())}" - filter_applications_query = f"{select_column_from_result_clause} where {filter_type_package_clause} and {filter_package_name_clause}" - - with self._use_role_optional(role): - self._sql_executor.execute_query(show_apps_query) - app_names = [ - app_row[0] - for app_row in self._sql_executor.execute_query( - filter_applications_query - ).fetchall() - ] - return app_names - # TODO move this to src/snowflake/cli/api/project/util.py in a separate # PR since it's codeowned by the CLI team From 99abebf67da8f79e67732669eb4638e1f98be0ed Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Mon, 2 Dec 2024 13:33:54 -0500 Subject: [PATCH 07/33] ws change --- src/snowflake/cli/_plugins/nativeapp/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index a1fba60e42..c8cf41bd7c 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -252,7 +252,6 @@ def app_teardown( if package_entity.identifier == app_package_entity.identifier ] - for app_entity in project.get_entities_by_type( ApplicationEntityModel.get_type() ).values(): From ce71ebfc04f654cc39cffd6d1a84ae6fec87543b Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 10:56:41 -0500 Subject: [PATCH 08/33] add schema to StagePathParts --- src/snowflake/cli/_plugins/stage/manager.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index dbeab38d94..b0aff983fa 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -41,7 +41,7 @@ from snowflake.cli.api.console import cli_console from snowflake.cli.api.constants import PYTHON_3_12 from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.util import to_string_literal +from snowflake.cli.api.project.util import extract_schema, to_string_literal from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.cli.api.stage_path import StagePath @@ -86,6 +86,10 @@ def path(self) -> str: def full_path(self) -> str: raise NotImplementedError + @property + def schema(self) -> str | None: + raise NotImplementedError + def replace_stage_prefix(self, file_path: str) -> str: raise NotImplementedError @@ -145,6 +149,10 @@ def path(self) -> str: def full_path(self) -> str: return f"{self.stage.rstrip('/')}/{self.directory}" + @property + def schema(self) -> str | None: + return extract_schema(self.stage) + def replace_stage_prefix(self, file_path: str) -> str: stage = Path(self.stage).parts[0] file_path_without_prefix = Path(file_path).parts[OMIT_FIRST] From 855b413ed22a0f95f790ccff8c538c68eed2b53f Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 11:40:22 -0500 Subject: [PATCH 09/33] use DefaultStagePathParts for stage in NA --- .../nativeapp/entities/application.py | 36 +++----- .../nativeapp/entities/application_package.py | 83 +++++++------------ .../cli/_plugins/nativeapp/sf_sql_facade.py | 12 +-- src/snowflake/cli/_plugins/stage/diff.py | 61 ++++++-------- src/snowflake/cli/_plugins/stage/manager.py | 10 ++- src/snowflake/cli/_plugins/stage/utils.py | 1 + src/snowflake/cli/api/entities/utils.py | 26 +++--- src/snowflake/cli/api/project/util.py | 14 ++-- 8 files changed, 105 insertions(+), 138 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application.py b/src/snowflake/cli/_plugins/nativeapp/entities/application.py index 8bb0cb3c4e..7ba283eeb4 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application.py @@ -30,7 +30,6 @@ COMMENT_COL, NAME_COL, OWNER_COL, - SOURCE_COL, SPECIAL_COMMENT, ) from snowflake.cli._plugins.nativeapp.entities.application_package import ( @@ -54,8 +53,8 @@ SameAccountInstallMethod, ) from snowflake.cli._plugins.nativeapp.sf_facade import get_snowflake_facade -from snowflake.cli._plugins.nativeapp.sf_sql_facade import _same_identifier from snowflake.cli._plugins.nativeapp.utils import needs_confirmation +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.workspace.context import ActionContext from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console.abc import AbstractConsole @@ -91,7 +90,6 @@ from snowflake.cli.api.project.schemas.updatable_model import DiscriminatorField from snowflake.cli.api.project.util import ( append_test_resource_suffix, - extract_schema, identifier_for_url, to_identifier, unquote_identifier, @@ -384,7 +382,8 @@ def action_deploy( package_entity: ApplicationPackageEntity = action_ctx.get_entity( self.package_entity_id ) - stage_fqn = package_entity.stage_fqn + # For now, from CLI's perspective, package owns the stage. This can change in the future. + stage_path = package_entity.stage_path if force: policy = AllowAlwaysPolicy() @@ -397,7 +396,7 @@ def action_deploy( if from_release_directive: self.create_or_upgrade_app( package=package_entity, - stage_fqn=stage_fqn, + stage_path=stage_path, install_method=SameAccountInstallMethod.release_directive(), policy=policy, interactive=interactive, @@ -419,7 +418,7 @@ def action_deploy( self.create_or_upgrade_app( package=package_entity, - stage_fqn=stage_fqn, + stage_path=stage_path, install_method=SameAccountInstallMethod.versioned_dev(version, patch), policy=policy, interactive=interactive, @@ -438,7 +437,7 @@ def action_deploy( ) self.create_or_upgrade_app( package=package_entity, - stage_fqn=stage_fqn, + stage_path=stage_path, install_method=SameAccountInstallMethod.unversioned_dev(), policy=policy, interactive=interactive, @@ -638,7 +637,7 @@ def get_objects_owned_by_application(self) -> List[ApplicationOwnedObject]: def create_or_upgrade_app( self, package: ApplicationPackageEntity, - stage_fqn: str, + stage_path: DefaultStagePathParts, install_method: SameAccountInstallMethod, policy: PolicyBase, interactive: bool, @@ -646,14 +645,10 @@ def create_or_upgrade_app( model = self._entity_model console = self._workspace_ctx.console debug_mode = model.debug - # PJ-TODO: in the future, the stage_fqn input to here could already have the subdir? - stage_fqn = stage_fqn or package.stage_fqn - stage_schema = extract_schema(stage_fqn) - path_to_artifacts = ( - f"{stage_fqn}/{package.stage_subdirectory}" - if package.stage_subdirectory - else stage_fqn - ) + + stage_fqn = stage_path.stage + stage_schema = stage_path.schema + sql_executor = get_sql_executor() with sql_executor.use_role(self.role): event_sharing = EventSharingHandler( @@ -676,18 +671,13 @@ def create_or_upgrade_app( app_role=self.role, show_app_row=show_app_row, ) - if not _same_identifier(show_app_row[SOURCE_COL], package.name): - # PJ- TODO: change to: are you sure you want to proceed? - raise ClickException( - "This application was not originally created from this package" - ) # If all the above checks are in order, proceed to upgrade try: console.step( f"Upgrading existing application object {self.name}." ) - using_clause = install_method.using_clause(path_to_artifacts) + using_clause = install_method.using_clause(stage_path.full_path) upgrade_cursor = sql_executor.execute_query( f"alter application {self.name} upgrade {using_clause}", ) @@ -784,7 +774,7 @@ def create_or_upgrade_app( ) authorize_telemetry_clause = f" AUTHORIZE_TELEMETRY_EVENT_SHARING = {str(new_authorize_event_sharing_value).upper()}" - using_clause = install_method.using_clause(path_to_artifacts) + using_clause = install_method.using_clause(stage_path.full_path) create_cursor = sql_executor.execute_query( dedent( f"""\ diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 5ec557850e..64012a9c16 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -47,7 +47,11 @@ ) from snowflake.cli._plugins.nativeapp.utils import needs_confirmation from snowflake.cli._plugins.stage.diff import DiffResult, compute_stage_diff -from snowflake.cli._plugins.stage.manager import StageManager +from snowflake.cli._plugins.stage.manager import ( + DefaultStagePathParts, + StageManager, + StagePathParts, +) from snowflake.cli._plugins.stage.utils import print_diff_to_console from snowflake.cli._plugins.workspace.context import ActionContext from snowflake.cli.api.cli_global_context import span @@ -79,7 +83,6 @@ from snowflake.cli.api.project.util import ( SCHEMA_AND_NAME, append_test_resource_suffix, - extract_schema, identifier_to_show_like_pattern, to_identifier, unquote_identifier, @@ -122,6 +125,7 @@ class ApplicationPackageEntityModel(EntityModelBase): title="Path to manifest.yml. Unused and deprecated starting with Snowflake CLI 3.2", default="", ) + # PJ-TODO: does it need sanitation? stage_subdirectory: Optional[str] = Field( title="Subfolder in stage", default="", @@ -167,10 +171,6 @@ def validate_source_stage(cls, input_value: str): return input_value -class SyncDataClass: - pass - - @attach_spans_to_entity_actions(entity_name="app_pkg") class ApplicationPackageEntity(EntityBase[ApplicationPackageEntityModel]): """ @@ -181,10 +181,6 @@ class ApplicationPackageEntity(EntityBase[ApplicationPackageEntityModel]): def project_root(self) -> Path: return self._workspace_ctx.project_root - @property - def stage_subdirectory(self) -> str: - return self._entity_model.stage_subdirectory - @property def deploy_root(self) -> Path: return ( @@ -218,16 +214,15 @@ def warehouse(self) -> str: ) or to_identifier(self._workspace_ctx.default_warehouse) @property - def stage_fqn(self) -> str: - return f"{self.name}.{self._entity_model.stage}" - - # @property - # def stage_root_path(self) -> str: - # return f"{self.stage_fqn}/{self.stage_subdirectory}" if self.stage_subdirectory else self.stage_fqn + def scratch_stage_path(self) -> DefaultStagePathParts: + return DefaultStagePathParts(f"{self.name}.{self._entity_model.scratch_stage}") @property - def scratch_stage_fqn(self) -> str: - return f"{self.name}.{self._entity_model.scratch_stage}" + def stage_path(self) -> DefaultStagePathParts: + stage_fqn = f"{self.name}.{self._entity_model.stage}" + subdir = self._entity_model.stage_subdirectory + full_path = f"{stage_fqn}/{subdir}" if subdir else stage_fqn + return DefaultStagePathParts(full_path) @property def post_deploy_hooks(self) -> list[PostDeployHook] | None: @@ -240,11 +235,11 @@ def action_bundle(self, action_ctx: ActionContext, *args, **kwargs): def action_diff( self, action_ctx: ActionContext, print_to_console: bool, *args, **kwargs ): + bundle_map = self._bundle() diff = compute_stage_diff( local_root=self.deploy_root, - stage_fqn=self.stage_fqn, - stage_subdirectory=self.stage_subdirectory, + stage_path=self.stage_path, ) if print_to_console: @@ -271,7 +266,7 @@ def action_deploy( paths=paths, print_diff=True, validate=validate, - stage_fqn=self.stage_fqn, + stage_path=self.stage_path, interactive=interactive, force=force, ) @@ -438,7 +433,7 @@ def action_version_create( paths=[], print_diff=True, validate=True, - stage_fqn=self.stage_fqn, + stage_path=self.stage_path, interactive=interactive, force=force, ) @@ -449,7 +444,6 @@ def action_version_create( self.get_existing_release_directive_info_for_version(resolved_version) ) except InsufficientPrivilegesError: - # PJ - TODO: insufficient privileges error will still show the "are you sure you want to create a new patch? prompt. it shouldn't" warning = ( "Could not check for existing release directives due to insufficient privileges. " "The MANAGE RELEASES privilege is required to check for existing release directives." @@ -602,7 +596,7 @@ def _deploy( paths: list[Path], print_diff: bool, validate: bool, - stage_fqn: str, + stage_path: StagePathParts, interactive: bool, force: bool, run_post_deploy_hooks: bool = True, @@ -617,7 +611,7 @@ def _deploy( policy = DenyAlwaysPolicy() console = workspace_ctx.console - stage_fqn = stage_fqn or self.stage_fqn + stage_path = stage_path or self.stage_path # 1. Create a bundle if one wasn't passed in bundle_map = bundle_map or self._bundle() @@ -632,21 +626,15 @@ def _deploy( with get_sql_executor().use_role(self.role): # 3. Upload files from deploy root local folder to the above stage - # PJ-TODO: move extract_schema in sync - stage_schema = extract_schema(stage_fqn) - # PJ-TODO: refactor stage into some object so we don't have to pass parts of it around. schema, subdir, fqn blah blah. like bundlemap. - # There might already be something like that for it diff = sync_deploy_root_with_stage( console=console, deploy_root=self.deploy_root, package_name=self.name, - stage_schema=stage_schema, bundle_map=bundle_map, role=self.role, prune=prune, recursive=recursive, - stage_fqn=stage_fqn, - stage_subdirectory=self.stage_subdirectory, + stage_path=stage_path, local_paths_to_sync=paths, print_diff=print_diff, ) @@ -724,9 +712,7 @@ def add_new_version(self, version: str, label: str | None = None) -> None: get_snowflake_facade().create_version_in_package( role=self.role, package_name=self.name, - stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" - if self.stage_subdirectory - else self.stage_fqn, + stage_path_to_artifacts=self.stage_path.full_path, version=version, label=label, ) @@ -750,9 +736,7 @@ def add_new_patch_to_version( new_patch = get_snowflake_facade().add_patch_to_package_version( role=self.role, package_name=self.name, - stage_fqn_with_subdir=f"{self.stage_fqn}/{self.stage_subdirectory}" - if self.stage_subdirectory - else self.stage_fqn, + stage_path_to_artifacts=self.stage_path.full_path, version=version, patch=patch, label=label, @@ -955,9 +939,9 @@ def get_validation_result( self, use_scratch_stage: bool, interactive: bool, force: bool ): """Call system$validate_native_app_setup() to validate deployed Native App setup script.""" - stage_fqn = self.stage_fqn + stage_path = self.stage_path if use_scratch_stage: - stage_fqn = self.scratch_stage_fqn + stage_path = self.scratch_stage_path self._deploy( bundle_map=None, prune=True, @@ -965,24 +949,19 @@ def get_validation_result( paths=[], print_diff=False, validate=False, - stage_fqn=self.scratch_stage_fqn, + stage_path=stage_path, interactive=interactive, force=force, run_post_deploy_hooks=False, ) - # prefixed_stage_fqn = StageManager.get_standard_stage_prefix(stage_fqn) - stage_fqn_with_subdir = ( - f"{stage_fqn}/{self.stage_subdirectory}" - if self.stage_subdirectory - else stage_fqn - ) - prefixed_stage_fqn_full = StageManager.get_standard_stage_prefix( - stage_fqn_with_subdir + prefixed_stage_fqn = StageManager.get_standard_stage_prefix( + stage_path.full_path ) + sql_executor = get_sql_executor() try: cursor = sql_executor.execute_query( - f"call system$validate_native_app_setup('{prefixed_stage_fqn_full}')" + f"call system$validate_native_app_setup('{prefixed_stage_fqn}')" ) except ProgrammingError as err: if err.errno == DOES_NOT_EXIST_OR_NOT_AUTHORIZED: @@ -995,11 +974,11 @@ def get_validation_result( finally: if use_scratch_stage: self._workspace_ctx.console.step( - f"Dropping stage {self.scratch_stage_fqn}." + f"Dropping stage {self.scratch_stage_path.stage}." ) with sql_executor.use_role(self.role): sql_executor.execute_query( - f"drop stage if exists {self.scratch_stage_fqn}" + f"drop stage if exists {self.scratch_stage_path.stage}" ) def resolve_version_info( diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index be505bc7a1..eab5ce40c7 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -192,7 +192,7 @@ def get_account_event_table(self, role: str | None = None) -> str | None: def create_version_in_package( self, package_name: str, - stage_fqn_with_subdir: str, + stage_path_to_artifacts: str, version: str, label: str | None = None, role: str | None = None, @@ -200,7 +200,7 @@ def create_version_in_package( """ Creates a new version in an existing application package. @param package_name: Name of the application package to alter. - @param stage_fqn_with_subdir: Stage fully qualified name. + @param stage_path_to_artifacts: Path to artifacts on the stage to create a version from. @param version: Version name to create. @param [Optional] role: Switch to this role while executing create version. @param [Optional] label: Label for this version, visible to consumers. @@ -217,7 +217,7 @@ def create_version_in_package( f"""\ alter application package {package_name} add version {version} - using @{stage_fqn_with_subdir}{with_label_cause} + using @{stage_path_to_artifacts}{with_label_cause} """ ) with self._use_role_optional(role): @@ -233,7 +233,7 @@ def create_version_in_package( def add_patch_to_package_version( self, package_name: str, - stage_fqn_with_subdir: str, + stage_path_to_artifacts: str, version: str, patch: int | None = None, label: str | None = None, @@ -242,7 +242,7 @@ def add_patch_to_package_version( """ Add a new patch, optionally a custom one, to an existing version in an application package. @param package_name: Name of the application package to alter. - @param stage_fqn_with_subdir: Stage fully qualified name. + @param stage_path_to_artifacts: Stage fully qualified name. @param version: Version name to create. @param [Optional] patch: Patch number to create. @param [Optional] label: Label for this patch, visible to consumers. @@ -263,7 +263,7 @@ def add_patch_to_package_version( f"""\ alter application package {package_name} add patch {patch_query} for version {version} - using @{stage_fqn_with_subdir}{with_label_clause} + using @{stage_path_to_artifacts}{with_label_clause} """ ) with self._use_role_optional(role): diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index 3fd6af1bcb..8715c86599 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -25,7 +25,7 @@ ) from snowflake.connector.cursor import DictCursor -from .manager import StageManager +from .manager import DefaultStagePathParts, StageManager from .md5 import UnknownMD5FormatError, file_matches_md5sum log = logging.getLogger(__name__) @@ -83,29 +83,24 @@ def enumerate_files(path: Path) -> List[Path]: return paths -def relative_to_stage_subdir(path: str, subdir: str | None = None) -> StagePathType: - path_parts = path.split("/") - # Remove stage name - path_parts.pop(0) - - path_wo_stage_name = StagePathType(*path_parts) - if subdir: - # Find file path relative to stage subdirectory - subdir_path = StagePathType(subdir) - return path_wo_stage_name.relative_to(subdir_path) - - return path_wo_stage_name +def relative_to_stage_path(path: str, stage_path: str) -> StagePathType: + """ + @param path: file path on the stage. + @param stage_path: root of stage. stage_name/[optionally/other/directories] + @return: path of file relative to the stage_path + """ + return StagePathType(path).relative_to(stage_path) def build_md5_map( - list_stage_cursor: DictCursor, stage_subdir: str | None = None + list_stage_cursor: DictCursor, stage_path: str ) -> Dict[StagePathType, Optional[str]]: """ - Returns a mapping of relative stage paths to their md5sums. + Returns a mapping of file paths to their md5sums. File paths are relative to the stage_path. """ all_files = list_stage_cursor.fetchall() return { - relative_to_stage_subdir(file["name"], stage_subdir): file["md5"] + relative_to_stage_path(file["name"], stage_path): file["md5"] for file in all_files } @@ -128,55 +123,51 @@ def preserve_from_diff( def compute_stage_diff( - local_root: Path, - stage_fqn: str, - stage_subdirectory: str | None = None, + local_root: Path, stage_path: DefaultStagePathParts ) -> DiffResult: """ - Diffs the files in a stage with a local folder. + Diffs the files in the local_root with files in the stage path that is stage_path_parts's full_path. """ - stage_fqn_with_subdir = ( - f"{stage_fqn}/{stage_subdirectory}" if stage_subdirectory else stage_fqn - ) stage_manager = StageManager() local_files = enumerate_files(local_root) - remote_files = stage_manager.list_files(stage_fqn_with_subdir) + remote_files = stage_manager.list_files(stage_path.full_path) - remote_md5 = build_md5_map(remote_files, stage_subdirectory) + # Create a mapping from remote_file path to file's md5sum. Path is relative to stage_name/directory. + remote_md5 = build_md5_map(remote_files, stage_path.path) result: DiffResult = DiffResult() for local_file in local_files: relpath = local_file.relative_to(local_root) - stage_path = to_stage_path(relpath) - if stage_path not in remote_md5: + rel_stage_path = to_stage_path(relpath) + if rel_stage_path not in remote_md5: # doesn't exist on the stage - result.only_local.append(stage_path) + result.only_local.append(rel_stage_path) else: # N.B. file size on stage is not always accurate, so cannot fail fast try: - if file_matches_md5sum(local_file, remote_md5[stage_path]): + if file_matches_md5sum(local_file, remote_md5[rel_stage_path]): # We are assuming that we will not get accidental collisions here due to the # large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance) # combined with the fact that the file name + path must also match elsewhere. - result.identical.append(stage_path) + result.identical.append(rel_stage_path) else: # either the file has changed, or we can't tell if it has - result.different.append(stage_path) + result.different.append(rel_stage_path) except UnknownMD5FormatError: log.warning( "Could not compare md5 for %s, assuming file has changed", local_file, exc_info=True, ) - result.different.append(stage_path) + result.different.append(rel_stage_path) # mark this file as seen - del remote_md5[stage_path] + del remote_md5[rel_stage_path] # every entry here is a file we never saw locally - for stage_path in remote_md5.keys(): - result.only_on_stage.append(stage_path) + for rel_stage_path in remote_md5.keys(): + result.only_on_stage.append(rel_stage_path) return result diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index dbeab38d94..b0aff983fa 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -41,7 +41,7 @@ from snowflake.cli.api.console import cli_console from snowflake.cli.api.constants import PYTHON_3_12 from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.util import to_string_literal +from snowflake.cli.api.project.util import extract_schema, to_string_literal from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.cli.api.stage_path import StagePath @@ -86,6 +86,10 @@ def path(self) -> str: def full_path(self) -> str: raise NotImplementedError + @property + def schema(self) -> str | None: + raise NotImplementedError + def replace_stage_prefix(self, file_path: str) -> str: raise NotImplementedError @@ -145,6 +149,10 @@ def path(self) -> str: def full_path(self) -> str: return f"{self.stage.rstrip('/')}/{self.directory}" + @property + def schema(self) -> str | None: + return extract_schema(self.stage) + def replace_stage_prefix(self, file_path: str) -> str: stage = Path(self.stage).parts[0] file_path_without_prefix = Path(file_path).parts[OMIT_FIRST] diff --git a/src/snowflake/cli/_plugins/stage/utils.py b/src/snowflake/cli/_plugins/stage/utils.py index a3c440f31b..4d9db5c3d9 100644 --- a/src/snowflake/cli/_plugins/stage/utils.py +++ b/src/snowflake/cli/_plugins/stage/utils.py @@ -9,6 +9,7 @@ from snowflake.cli.api.console import cli_console as cc +# PJ-TODO: update to add subdirectory to print def print_diff_to_console( diff: DiffResult, bundle_map: Optional[BundleMap] = None, diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 46af1de0a7..6db2029664 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -22,6 +22,7 @@ sync_local_diff_with_stage, to_stage_path, ) +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.stage.utils import print_diff_to_console from snowflake.cli.api.cli_global_context import get_cli_context, span from snowflake.cli.api.console.abc import AbstractConsole @@ -81,13 +82,11 @@ def sync_deploy_root_with_stage( console: AbstractConsole, deploy_root: Path, package_name: str, - stage_schema: str, bundle_map: BundleMap, role: str, prune: bool, recursive: bool, - stage_fqn: str, - stage_subdirectory: str | None = None, + stage_path: DefaultStagePathParts, local_paths_to_sync: List[Path] | None = None, print_diff: bool = True, ) -> DiffResult: @@ -100,39 +99,36 @@ def sync_deploy_root_with_stage( role (str): The name of the role to use for queries and commands. prune (bool): Whether to prune artifacts from the stage that don't exist locally. recursive (bool): Whether to traverse directories recursively. - stage_fqn (str): The name of the stage to diff against and upload to. + stage_path (DefaultStagePathParts): stage path object. + local_paths_to_sync (List[Path], optional): List of local paths to sync. Defaults to None to sync all - local paths. Note that providing an empty list here is equivalent to None. + local paths. Note that providing an empty list here is equivalent to None. print_diff (bool): Whether to print the diff between the local files and the remote stage. Defaults to True Returns: A `DiffResult` instance describing the changes that were performed. """ sql_facade = get_snowflake_facade() + schema = stage_path.schema + stage_fqn = stage_path.stage # Does a stage already exist within the application package, or we need to create one? # Using "if not exists" should take care of either case. console.step( f"Checking if stage {stage_fqn} exists, or creating a new one if none exists." ) if not sql_facade.stage_exists(stage_fqn): - sql_facade.create_schema(stage_schema, database=package_name) + sql_facade.create_schema(schema, database=package_name) sql_facade.create_stage(stage_fqn) # Perform a diff operation and display results to the user for informational purposes - # PJ - TODO: make optional / - # PJ - rename this - stage_fqn_with_subdir = ( - f"{stage_fqn}/{stage_subdirectory}" if stage_subdirectory else stage_fqn - ) if print_diff: console.step( - f"Performing a diff between the Snowflake stage {stage_fqn_with_subdir} and your local deploy_root {deploy_root.resolve()} directory." + f"Performing a diff between the Snowflake stage {stage_path.path} and your local deploy_root {deploy_root.resolve()} directory." ) diff: DiffResult = compute_stage_diff( local_root=deploy_root, - stage_fqn=stage_fqn, - stage_subdirectory=stage_subdirectory, + stage_path=stage_path, ) if local_paths_to_sync: @@ -194,7 +190,7 @@ def sync_deploy_root_with_stage( role=role, deploy_root_path=deploy_root, diff_result=diff, - stage_fqn=stage_fqn_with_subdir, + stage_fqn=stage_fqn, ) return diff diff --git a/src/snowflake/cli/api/project/util.py b/src/snowflake/cli/api/project/util.py index dcb1c50b8f..019e3a8c5b 100644 --- a/src/snowflake/cli/api/project/util.py +++ b/src/snowflake/cli/api/project/util.py @@ -20,6 +20,8 @@ from typing import List, Optional from urllib.parse import quote +# from snowflake.cli._plugins.stage.manager import StagePathParts + IDENTIFIER = r'((?:"[^"]*(?:""[^"]*)*")|(?:[A-Za-z_][\w$]{0,254}))' IDENTIFIER_NO_LENGTH = r'((?:"[^"]*(?:""[^"]*)*")|(?:[A-Za-z_][\w$]*))' DB_SCHEMA_AND_NAME = f"{IDENTIFIER}[.]{IDENTIFIER}[.]{IDENTIFIER}" @@ -194,19 +196,19 @@ def extract_schema(qualified_name: str): return None -# def extract_schema_from_path_to_version_directory(path: str) -> str: +# def extract_schema_from_full_stage_path(path: str) -> str: # """ -# Extracts the schema from a path to a version directory. -# Path to version directory is a stage fqn with an optional subdirectory. It can take any of these forms: +# Extracts the schema from a full stage path. +# Full stage path is a stage fqn with optional subdirectories: # db.schema.stage # db.schema.stage/subdirectory # schema.stage # schema.stage/subdirectory # """ -# DB_SCHEMA_AND_NAME = r"(?P[^.]+)\.(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" -# SCHEMA_AND_NAME = r"(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" +# DB_SCHEMA_STAGE_PATTERN = r"(?P[^.]+)\.(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" +# SCHEMA_STAGE_PATTERN = r"(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" # -# match = re.match(DB_SCHEMA_AND_NAME, path) or re.match(SCHEMA_AND_NAME, path) +# match = re.match(DB_SCHEMA_STAGE_PATTERN, path) or re.match(SCHEMA_STAGE_PATTERN, path) # if match: # return match.groupdict()["schema"] # return None From 68904aaae44ddfc8a439ad355a959bdbdde7d877 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 11:45:30 -0500 Subject: [PATCH 10/33] remmove comments --- .../nativeapp/entities/application.py | 2 +- .../cli/_plugins/nativeapp/sf_sql_facade.py | 3 +-- src/snowflake/cli/_plugins/stage/utils.py | 1 - .../cli/_plugins/workspace/manager.py | 17 ---------------- src/snowflake/cli/api/project/util.py | 20 ------------------- 5 files changed, 2 insertions(+), 41 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application.py b/src/snowflake/cli/_plugins/nativeapp/entities/application.py index 7ba283eeb4..0335645576 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application.py @@ -382,7 +382,7 @@ def action_deploy( package_entity: ApplicationPackageEntity = action_ctx.get_entity( self.package_entity_id ) - # For now, from CLI's perspective, package owns the stage. This can change in the future. + stage_path = package_entity.stage_path if force: diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index eab5ce40c7..6f44b8235a 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -229,7 +229,6 @@ def create_version_in_package( f"Failed to add version {version} to application package {package_name}.", ) - # PJ-TODO: rename stage_fqn_with_subdir everywhere to a better name like stage_path_to_artifacts def add_patch_to_package_version( self, package_name: str, @@ -242,7 +241,7 @@ def add_patch_to_package_version( """ Add a new patch, optionally a custom one, to an existing version in an application package. @param package_name: Name of the application package to alter. - @param stage_path_to_artifacts: Stage fully qualified name. + @param stage_path_to_artifacts: Path to artifacts on the stage to create a version from. @param version: Version name to create. @param [Optional] patch: Patch number to create. @param [Optional] label: Label for this patch, visible to consumers. diff --git a/src/snowflake/cli/_plugins/stage/utils.py b/src/snowflake/cli/_plugins/stage/utils.py index 4d9db5c3d9..a3c440f31b 100644 --- a/src/snowflake/cli/_plugins/stage/utils.py +++ b/src/snowflake/cli/_plugins/stage/utils.py @@ -9,7 +9,6 @@ from snowflake.cli.api.console import cli_console as cc -# PJ-TODO: update to add subdirectory to print def print_diff_to_console( diff: DiffResult, bundle_map: Optional[BundleMap] = None, diff --git a/src/snowflake/cli/_plugins/workspace/manager.py b/src/snowflake/cli/_plugins/workspace/manager.py index 75e47604b9..05d688ea54 100644 --- a/src/snowflake/cli/_plugins/workspace/manager.py +++ b/src/snowflake/cli/_plugins/workspace/manager.py @@ -53,23 +53,6 @@ def get_entity(self, entity_id: str): self._entities_cache[entity_id] = entity_cls(entity_model, workspace_ctx) return self._entities_cache[entity_id] - # def get_entities_with_identifier(self, identifier: str): - # if identifier in self._entities_identifier_cache: - # return self._entities_cache[entity_id] - # entity_model = self._project_definition.entities.get(entity_id, None) - # if entity_model is None: - # raise ValueError(f"No such entity ID: {entity_id}") - # entity_model_cls = entity_model.__class__ - # entity_cls = v2_entity_model_to_entity_map[entity_model_cls] - # workspace_ctx = WorkspaceContext( - # console=cc, - # project_root=self.project_root, - # get_default_role=_get_default_role, - # get_default_warehouse=_get_default_warehouse, - # ) - # self._entities_cache[entity_id] = entity_cls(entity_model, workspace_ctx) - # return self._entities_cache[entity_id] - def perform_action(self, entity_id: str, action: EntityActions, *args, **kwargs): """ Instantiates an entity of the given ID and calls the given action on it. diff --git a/src/snowflake/cli/api/project/util.py b/src/snowflake/cli/api/project/util.py index 019e3a8c5b..564ebbd867 100644 --- a/src/snowflake/cli/api/project/util.py +++ b/src/snowflake/cli/api/project/util.py @@ -20,8 +20,6 @@ from typing import List, Optional from urllib.parse import quote -# from snowflake.cli._plugins.stage.manager import StagePathParts - IDENTIFIER = r'((?:"[^"]*(?:""[^"]*)*")|(?:[A-Za-z_][\w$]{0,254}))' IDENTIFIER_NO_LENGTH = r'((?:"[^"]*(?:""[^"]*)*")|(?:[A-Za-z_][\w$]*))' DB_SCHEMA_AND_NAME = f"{IDENTIFIER}[.]{IDENTIFIER}[.]{IDENTIFIER}" @@ -196,24 +194,6 @@ def extract_schema(qualified_name: str): return None -# def extract_schema_from_full_stage_path(path: str) -> str: -# """ -# Extracts the schema from a full stage path. -# Full stage path is a stage fqn with optional subdirectories: -# db.schema.stage -# db.schema.stage/subdirectory -# schema.stage -# schema.stage/subdirectory -# """ -# DB_SCHEMA_STAGE_PATTERN = r"(?P[^.]+)\.(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" -# SCHEMA_STAGE_PATTERN = r"(?P[^.]+)\.(?P[^/]+)(?:/(?P.+))?" -# -# match = re.match(DB_SCHEMA_STAGE_PATTERN, path) or re.match(SCHEMA_STAGE_PATTERN, path) -# if match: -# return match.groupdict()["schema"] -# return None - - def first_set_env(*keys: str): for k in keys: v = os.getenv(k) From 5fc642e6ed14f8b582e9b8d27d84629caced754b Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 14:33:35 -0500 Subject: [PATCH 11/33] clean up --- src/snowflake/cli/_plugins/stage/diff.py | 33 ++++++++++++------- .../cli/_plugins/workspace/manager.py | 1 - src/snowflake/cli/api/entities/utils.py | 2 +- tests/stage/test_diff.py | 4 +-- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index 8715c86599..fbd108f69f 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -194,7 +194,7 @@ def to_local_path(stage_path: StagePathType) -> Path: def delete_only_on_stage_files( stage_manager: StageManager, - stage_fqn: str, + stage_root: str, only_on_stage: List[StagePathType], role: Optional[str] = None, ): @@ -202,12 +202,12 @@ def delete_only_on_stage_files( Deletes all files from a Snowflake stage according to the input list of filenames, using a custom role. """ for _stage_path in only_on_stage: - stage_manager.remove(stage_name=stage_fqn, path=str(_stage_path), role=role) + stage_manager.remove(stage_name=stage_root, path=str(_stage_path), role=role) def put_files_on_stage( stage_manager: StageManager, - stage_fqn: str, + stage_root: str, deploy_root_path: Path, stage_paths: List[StagePathType], role: Optional[str] = None, @@ -219,7 +219,9 @@ def put_files_on_stage( for _stage_path in stage_paths: stage_sub_path = get_stage_subpath(_stage_path) full_stage_path = ( - f"{stage_fqn}/{stage_sub_path}" if stage_sub_path else stage_fqn + f"{stage_root.rstrip('/')}/{stage_sub_path}" + if stage_sub_path + else stage_root ) stage_manager.put( local_path=deploy_root_path / to_local_path(_stage_path), @@ -230,7 +232,10 @@ def put_files_on_stage( def sync_local_diff_with_stage( - role: str | None, deploy_root_path: Path, diff_result: DiffResult, stage_fqn: str + role: str | None, + deploy_root_path: Path, + diff_result: DiffResult, + stage_full_path: str, ): """ Syncs a given local directory's contents with a Snowflake stage, including removing old files, and re-uploading modified and new files. @@ -243,18 +248,22 @@ def sync_local_diff_with_stage( try: delete_only_on_stage_files( - stage_manager, stage_fqn, diff_result.only_on_stage, role + stage_manager, stage_full_path, diff_result.only_on_stage, role ) put_files_on_stage( - stage_manager, - stage_fqn, - deploy_root_path, - diff_result.different, - role, + stage_manager=stage_manager, + stage_root=stage_full_path, + deploy_root_path=deploy_root_path, + stage_paths=diff_result.different, + role=role, overwrite=True, ) put_files_on_stage( - stage_manager, stage_fqn, deploy_root_path, diff_result.only_local, role + stage_manager=stage_manager, + stage_root=stage_full_path, + deploy_root_path=deploy_root_path, + stage_paths=diff_result.only_local, + role=role, ) except Exception as err: # Could be ProgrammingError or IntegrityError from SnowflakeCursor diff --git a/src/snowflake/cli/_plugins/workspace/manager.py b/src/snowflake/cli/_plugins/workspace/manager.py index 05d688ea54..25b56d542f 100644 --- a/src/snowflake/cli/_plugins/workspace/manager.py +++ b/src/snowflake/cli/_plugins/workspace/manager.py @@ -31,7 +31,6 @@ def __init__(self, project_definition: ProjectDefinition, project_root: Path): self._entities_cache: Dict[str, Entity] = {} self._project_definition: DefinitionV20 = project_definition self._project_root = project_root - self._entities_identifier_cache: Dict[str, list[Entity]] = {} def get_entity(self, entity_id: str): """ diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 6db2029664..d1a2b5687d 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -190,7 +190,7 @@ def sync_deploy_root_with_stage( role=role, deploy_root_path=deploy_root, diff_result=diff, - stage_fqn=stage_fqn, + stage_full_path=stage_path.full_path, ) return diff diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index 8ee1e462fd..6b906272bd 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -252,7 +252,7 @@ def test_put_files_on_stage(mock_put, overwrite_param): ) as local_path: put_files_on_stage( stage_manager=StageManager(), - stage_fqn=stage_name, + stage_root=stage_name, deploy_root_path=local_path, stage_paths=as_stage_paths(["ui/nested/environment.yml", "README.md"]), role="some_role", @@ -306,7 +306,7 @@ def test_sync_local_diff_with_stage(mock_remove, other_directory): role="some_role", deploy_root_path=temp_dir, diff_result=diff, - stage_fqn=stage_name, + stage_full_path=stage_name, ) From 49275b9243d6fdad7cd43729969fccbef932327c Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 14:33:35 -0500 Subject: [PATCH 12/33] clean up --- .gitignore | 1 - src/snowflake/cli/_plugins/nativeapp/commands.py | 5 ----- 2 files changed, 6 deletions(-) diff --git a/.gitignore b/.gitignore index fbac1aaaf9..cdc3b6490a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ gen_docs/ .env .vscode tmp/ -py11venv/ ^app.zip ^snowflake.yml diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index c8cf41bd7c..1ef718f0f8 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -21,7 +21,6 @@ from textwrap import dedent from typing import Generator, Iterable, List, Optional, cast -import pydevd_pycharm import typer from snowflake.cli._plugins.nativeapp.common_flags import ( ForceOption, @@ -57,10 +56,6 @@ ) from typing_extensions import Annotated -pydevd_pycharm.settrace( - "localhost", port=12345, stdoutToServer=True, stderrToServer=True, suspend=False -) - app = SnowTyperFactory( name="app", help="Manages a Snowflake Native App", From 04d807e6b3c196b8c866ee6aec39b6ba5476f66b Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 15:05:32 -0500 Subject: [PATCH 13/33] update comment --- src/snowflake/cli/_plugins/nativeapp/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index 1ef718f0f8..a054f3af3b 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -250,6 +250,7 @@ def app_teardown( for app_entity in project.get_entities_by_type( ApplicationEntityModel.get_type() ).values(): + # Drop each app if app_entity.from_.target in all_packages_with_id: ws.perform_action( app_entity.entity_id, @@ -258,7 +259,6 @@ def app_teardown( interactive=interactive, cascade=cascade, ) - # Then drop the package ws.perform_action( app_package_entity.entity_id, From ff369e7c367b97794d83fc8dee7527cd92fcf768 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 15:22:49 -0500 Subject: [PATCH 14/33] rename stage --- .../cli/_plugins/nativeapp/sf_sql_facade.py | 6 +++--- tests/nativeapp/test_sf_sql_facade.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index eca72322d0..70aceea843 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -579,7 +579,7 @@ def upgrade_application( self, name: str, install_method: SameAccountInstallMethod, - stage_fqn: str, + stage_path_to_artifacts: str, role: str, warehouse: str, debug_mode: bool | None, @@ -590,7 +590,7 @@ def upgrade_application( @param name: Name of the application object @param install_method: Method of installing the application - @param stage_fqn: FQN of the stage housing the application artifacts + @param stage_path_to_artifacts: FQN of the stage housing the application artifacts @param role: Role to use when creating the application and provider-side objects @param warehouse: Warehouse which is required to create an application object @param debug_mode: Whether to enable debug mode; None means not explicitly enabled or disabled @@ -605,7 +605,7 @@ def upgrade_application( with self._use_role_optional(role), self._use_warehouse_optional(warehouse): try: - using_clause = install_method.using_clause(stage_fqn) + using_clause = install_method.using_clause(stage_path_to_artifacts) upgrade_cursor = self._sql_executor.execute_query( f"alter application {name} upgrade {using_clause}", ) diff --git a/tests/nativeapp/test_sf_sql_facade.py b/tests/nativeapp/test_sf_sql_facade.py index f1d3de9bb2..fbcb4ee3d1 100644 --- a/tests/nativeapp/test_sf_sql_facade.py +++ b/tests/nativeapp/test_sf_sql_facade.py @@ -1796,7 +1796,7 @@ def test_upgrade_application_unversioned( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, @@ -1847,7 +1847,7 @@ def test_upgrade_application_version_and_patch( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.versioned_dev("3", 2), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -1894,7 +1894,7 @@ def test_upgrade_application_from_release_directive( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -1940,7 +1940,7 @@ def test_upgrade_application_converts_expected_programmingerrors_to_user_errors( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -2005,7 +2005,7 @@ def test_upgrade_application_special_message_for_event_sharing_error( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.versioned_dev("v1"), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=False, should_authorize_event_sharing=False, role=role, @@ -2054,7 +2054,7 @@ def test_upgrade_application_converts_unexpected_programmingerrors_to_unclassifi sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, From 31e1f704c783829c43b47623157b86be968eb456 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 15:53:52 -0500 Subject: [PATCH 15/33] remove trailing slash when no directory --- src/snowflake/cli/_plugins/stage/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index b0aff983fa..71b4e18cdd 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -143,11 +143,11 @@ def __init__(self, stage_path: str): @property def path(self) -> str: - return f"{self.stage_name.rstrip('/')}/{self.directory}" + return f"{self.stage_name.rstrip('/')}/{self.directory}".rstrip("/") @property def full_path(self) -> str: - return f"{self.stage.rstrip('/')}/{self.directory}" + return f"{self.stage.rstrip('/')}/{self.directory}".rstrip("/") @property def schema(self) -> str | None: @@ -201,7 +201,7 @@ def path(self) -> str: @property def full_path(self) -> str: - return f"{self.stage}/{self.directory}" + return f"{self.stage}/{self.directory}".rstrip("/") def replace_stage_prefix(self, file_path: str) -> str: if Path(file_path).parts[0] == self.stage_name: From f52c6cbec296615e846feafb0ae08bab6a2e04e7 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 16:30:26 -0500 Subject: [PATCH 16/33] update unit tests --- .../nativeapp/entities/application.py | 2 +- .../cli/_plugins/nativeapp/sf_sql_facade.py | 8 +-- src/snowflake/cli/_plugins/stage/diff.py | 4 +- src/snowflake/cli/_plugins/stage/manager.py | 4 +- .../test_application_package_entity.py | 4 +- tests/nativeapp/test_event_sharing.py | 7 ++- tests/nativeapp/test_manager.py | 20 ++++--- tests/nativeapp/test_run_processor.py | 59 ++++++++++--------- tests/nativeapp/test_sf_sql_facade.py | 10 ++-- tests/nativeapp/test_version_create.py | 8 +-- tests/stage/test_diff.py | 19 +++--- 11 files changed, 76 insertions(+), 69 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application.py b/src/snowflake/cli/_plugins/nativeapp/entities/application.py index d1755bde56..4e74fe496b 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application.py @@ -666,7 +666,7 @@ def _create_app( name=self.name, package_name=package.name, install_method=install_method, - stage_fqn=stage_path.full_path, + stage_path_to_artifacts=stage_path.full_path, debug_mode=self.debug, should_authorize_event_sharing=event_sharing.should_authorize_event_sharing(), role=self.role, diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index 70aceea843..2cf9c6cb14 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -590,7 +590,7 @@ def upgrade_application( @param name: Name of the application object @param install_method: Method of installing the application - @param stage_path_to_artifacts: FQN of the stage housing the application artifacts + @param stage_path_to_artifacts: Path to directory in stage housing the application artifacts @param role: Role to use when creating the application and provider-side objects @param warehouse: Warehouse which is required to create an application object @param debug_mode: Whether to enable debug mode; None means not explicitly enabled or disabled @@ -675,7 +675,7 @@ def create_application( name: str, package_name: str, install_method: SameAccountInstallMethod, - stage_fqn: str, + stage_path_to_artifacts: str, role: str, warehouse: str, debug_mode: bool | None, @@ -688,7 +688,7 @@ def create_application( @param name: Name of the application object @param package_name: Name of the application package to install the application from @param install_method: Method of installing the application - @param stage_fqn: FQN of the stage housing the application artifacts + @param stage_path_to_artifacts: Path to directory in stage housing the application artifacts @param role: Role to use when creating the application and provider-side objects @param warehouse: Warehouse which is required to create an application object @param debug_mode: Whether to enable debug mode; None means not explicitly enabled or disabled @@ -710,7 +710,7 @@ def create_application( ) authorize_telemetry_clause = f" AUTHORIZE_TELEMETRY_EVENT_SHARING = {str(should_authorize_event_sharing).upper()}" - using_clause = install_method.using_clause(stage_fqn) + using_clause = install_method.using_clause(stage_path_to_artifacts) with self._use_role_optional(role), self._use_warehouse_optional(warehouse): try: create_cursor = self._sql_executor.execute_query( diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index fbd108f69f..2cd498ca84 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -219,9 +219,7 @@ def put_files_on_stage( for _stage_path in stage_paths: stage_sub_path = get_stage_subpath(_stage_path) full_stage_path = ( - f"{stage_root.rstrip('/')}/{stage_sub_path}" - if stage_sub_path - else stage_root + f"{stage_root}/{stage_sub_path}" if stage_sub_path else stage_root ) stage_manager.put( local_path=deploy_root_path / to_local_path(_stage_path), diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index b0aff983fa..40d5ea86d1 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -143,11 +143,11 @@ def __init__(self, stage_path: str): @property def path(self) -> str: - return f"{self.stage_name.rstrip('/')}/{self.directory}" + return f"{self.stage_name.rstrip('/')}/{self.directory}".rstrip("/") @property def full_path(self) -> str: - return f"{self.stage.rstrip('/')}/{self.directory}" + return f"{self.stage.rstrip('/')}/{self.directory}".rstrip("/") @property def schema(self) -> str | None: diff --git a/tests/nativeapp/test_application_package_entity.py b/tests/nativeapp/test_application_package_entity.py index 3966a391ff..b60f7474b1 100644 --- a/tests/nativeapp/test_application_package_entity.py +++ b/tests/nativeapp/test_application_package_entity.py @@ -25,6 +25,7 @@ ApplicationPackageEntity, ApplicationPackageEntityModel, ) +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.workspace.context import ActionContext, WorkspaceContext from snowflake.connector.cursor import DictCursor @@ -153,12 +154,11 @@ def test_deploy( app_pkg._workspace_ctx.project_root / Path("output/deploy") # noqa SLF001 ), package_name="pkg", - stage_schema="app_src", bundle_map=mock.ANY, role="app_role", prune=False, recursive=False, - stage_fqn="pkg.app_src.stage", + stage_path=DefaultStagePathParts("pkg.app_src.stage"), local_paths_to_sync=["a/b", "c"], print_diff=True, ) diff --git a/tests/nativeapp/test_event_sharing.py b/tests/nativeapp/test_event_sharing.py index 463e58719e..c3abbf615c 100644 --- a/tests/nativeapp/test_event_sharing.py +++ b/tests/nativeapp/test_event_sharing.py @@ -37,6 +37,7 @@ SameAccountInstallMethod, ) from snowflake.cli._plugins.nativeapp.sf_facade_exceptions import UserInputError +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.workspace.context import ActionContext, WorkspaceContext from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.console.abc import AbstractConsole @@ -151,7 +152,7 @@ def _create_or_upgrade_app( return app.create_or_upgrade_app( package=pkg, - stage_fqn=stage_fqn, + stage_path=DefaultStagePathParts(stage_fqn), install_method=install_method, policy=policy, interactive=is_interactive, @@ -297,7 +298,7 @@ def _setup_mocks_for_create_app( install_method=SameAccountInstallMethod.release_directive() if is_prod else SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=None, should_authorize_event_sharing=expected_authorize_telemetry_flag, role="app_role", @@ -400,7 +401,7 @@ def _setup_mocks_for_upgrade_app( install_method=SameAccountInstallMethod.release_directive() if is_prod else SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=None, should_authorize_event_sharing=expected_authorize_telemetry_flag, role="app_role", diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index 437bede8fb..4c146da8a6 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -51,6 +51,7 @@ DiffResult, StagePathType, ) +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.workspace.manager import WorkspaceManager from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.entities.common import EntityActions @@ -143,12 +144,11 @@ def test_sync_deploy_root_with_stage( console=cc, deploy_root=dm.project_root / pkg_model.deploy_root, package_name=package_name, - stage_schema=stage_schema, bundle_map=mock_bundle_map, role="new_role", prune=True, recursive=True, - stage_fqn=stage_fqn, + stage_path=DefaultStagePathParts(stage_fqn), ) mock_stage_exists.assert_called_once_with(stage_fqn) @@ -156,13 +156,14 @@ def test_sync_deploy_root_with_stage( mock_create_schema.assert_called_once_with(stage_schema, database=package_name) mock_create_stage.assert_called_once_with(stage_fqn) mock_compute_stage_diff.assert_called_once_with( - dm.project_root / pkg_model.deploy_root, "app_pkg.app_src.stage" + local_root=dm.project_root / pkg_model.deploy_root, + stage_path=DefaultStagePathParts("app_pkg.app_src.stage"), ) mock_local_diff_with_stage.assert_called_once_with( role="new_role", deploy_root_path=dm.project_root / pkg_model.deploy_root, diff_result=mock_diff_result, - stage_fqn="app_pkg.app_src.stage", + stage_full_path="app_pkg.app_src.stage", ) @@ -213,12 +214,11 @@ def test_sync_deploy_root_with_stage_prune( console=mock_console, deploy_root=dm.project_root / pkg_model.deploy_root, package_name=package_name, - stage_schema=extract_schema(stage_fqn), bundle_map=mock_bundle_map, role="new_role", prune=prune, recursive=True, - stage_fqn=stage_fqn, + stage_path=DefaultStagePathParts(stage_fqn), ) if expected_warn: @@ -1329,7 +1329,9 @@ def test_validate_use_scratch_stage(mock_execute, mock_deploy, temp_dir, mock_cu paths=[], print_diff=False, validate=False, - stage_fqn=f"{pkg_model.fqn.name}.{pkg_model.scratch_stage}", + stage_path=DefaultStagePathParts( + f"{pkg_model.fqn.name}.{pkg_model.scratch_stage}" + ), interactive=False, force=True, run_post_deploy_hooks=False, @@ -1405,7 +1407,9 @@ def test_validate_failing_drops_scratch_stage( paths=[], print_diff=False, validate=False, - stage_fqn=f"{pkg_model.fqn.name}.{pkg_model.scratch_stage}", + stage_path=DefaultStagePathParts( + f"{pkg_model.fqn.name}.{pkg_model.scratch_stage}" + ), interactive=False, force=True, run_post_deploy_hooks=False, diff --git a/tests/nativeapp/test_run_processor.py b/tests/nativeapp/test_run_processor.py index dc6df30b4b..7968ee0300 100644 --- a/tests/nativeapp/test_run_processor.py +++ b/tests/nativeapp/test_run_processor.py @@ -54,6 +54,7 @@ UserInputError, ) from snowflake.cli._plugins.stage.diff import DiffResult +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.workspace.context import ActionContext, WorkspaceContext from snowflake.cli._plugins.workspace.manager import WorkspaceManager from snowflake.cli.api.console import cli_console as cc @@ -165,7 +166,7 @@ def _create_or_upgrade_app( return app.create_or_upgrade_app( package=pkg, - stage_fqn=stage_fqn, + stage_path=DefaultStagePathParts(stage_fqn), install_method=install_method, policy=policy, interactive=interactive, @@ -269,7 +270,7 @@ def test_create_dev_app_w_warehouse_access_exception( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -340,7 +341,7 @@ def test_create_dev_app_create_new_w_no_additional_privileges( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -413,7 +414,7 @@ def test_create_or_upgrade_dev_app_with_warning( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -427,7 +428,7 @@ def test_create_or_upgrade_dev_app_with_warning( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -481,7 +482,7 @@ def test_create_dev_app_create_new_with_additional_privileges( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -558,7 +559,7 @@ def test_create_dev_app_create_new_w_missing_warehouse_exception( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -669,7 +670,7 @@ def test_create_dev_app_incorrect_owner( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -722,7 +723,7 @@ def test_create_dev_app_no_diff_changes( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -778,7 +779,7 @@ def test_create_dev_app_w_diff_changes( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -902,7 +903,7 @@ def test_create_dev_app_create_new_quoted( name='"My Application"', package_name='"My Package"', install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn='"My Package".app_src.stage', + stage_path_to_artifacts='"My Package".app_src.stage', debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -959,7 +960,7 @@ def test_create_dev_app_create_new_quoted_override( name='"My Application"', package_name='"My Package"', install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn='"My Package".app_src.stage', + stage_path_to_artifacts='"My Package".app_src.stage', debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1041,7 +1042,7 @@ def test_create_dev_app_recreate_app_when_orphaned( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1053,7 +1054,7 @@ def test_create_dev_app_recreate_app_when_orphaned( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1180,7 +1181,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1193,7 +1194,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1317,7 +1318,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade_unknown_obje mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1329,7 +1330,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade_unknown_obje name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1474,7 +1475,7 @@ def test_upgrade_app_incorrect_owner( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1529,7 +1530,7 @@ def test_upgrade_app_succeeds( mock_sql_facade_upgrade_application.assert_called_once_with( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1588,7 +1589,7 @@ def test_upgrade_app_fails_generic_error( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1669,7 +1670,7 @@ def test_upgrade_app_fails_upgrade_restriction_error( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1749,7 +1750,7 @@ def test_versioned_app_upgrade_to_unversioned( mock_sql_facade_upgrade_application.assert_called_once_with( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1760,7 +1761,7 @@ def test_versioned_app_upgrade_to_unversioned( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1868,7 +1869,7 @@ def test_upgrade_app_fails_drop_fails( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1949,7 +1950,7 @@ def test_upgrade_app_recreate_app( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1961,7 +1962,7 @@ def test_upgrade_app_recreate_app( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2130,7 +2131,7 @@ def test_upgrade_app_recreate_app_from_version( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.versioned_dev("v1"), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2142,7 +2143,7 @@ def test_upgrade_app_recreate_app_from_version( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.versioned_dev("v1"), - stage_fqn=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, diff --git a/tests/nativeapp/test_sf_sql_facade.py b/tests/nativeapp/test_sf_sql_facade.py index fbcb4ee3d1..9d87bc5788 100644 --- a/tests/nativeapp/test_sf_sql_facade.py +++ b/tests/nativeapp/test_sf_sql_facade.py @@ -2105,7 +2105,7 @@ def test_create_application_with_minimal_clauses( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, @@ -2154,7 +2154,7 @@ def test_create_application_with_all_clauses( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -2207,7 +2207,7 @@ def test_create_application_converts_expected_programmingerrors_to_user_errors( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, @@ -2266,7 +2266,7 @@ def test_create_application_special_message_for_event_sharing_error( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.versioned_dev("3", 1), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=False, should_authorize_event_sharing=False, role=role, @@ -2324,7 +2324,7 @@ def test_create_application_converts_unexpected_programmingerrors_to_unclassifie name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + stage_path_to_artifacts=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, diff --git a/tests/nativeapp/test_version_create.py b/tests/nativeapp/test_version_create.py index e035d50b65..0e4c689bd2 100644 --- a/tests/nativeapp/test_version_create.py +++ b/tests/nativeapp/test_version_create.py @@ -699,7 +699,7 @@ def test_manifest_version_info_not_used( mock_create_version.assert_called_with( role=role, package_name="app_pkg", - stage_fqn=f"app_pkg.{stage}", + stage_path_to_artifacts=f"app_pkg.{stage}", version=version_cli, label="", ) @@ -765,7 +765,7 @@ def test_manifest_patch_is_not_used( mock_create_patch.assert_called_with( role=role, package_name="app_pkg", - stage_fqn=f"app_pkg.{stage}", + stage_path_to_artifacts=f"app_pkg.{stage}", version=version_cli, patch=patch, # ensure empty label is used to replace label from manifest.yml @@ -840,7 +840,7 @@ def test_version_from_manifest( mock_create_patch.assert_called_with( role=role, package_name="app_pkg", - stage_fqn=f"app_pkg.{stage}", + stage_path_to_artifacts=f"app_pkg.{stage}", version="manifest_version", patch=manifest_patch, label=cli_label if cli_label is not None else manifest_label, @@ -914,7 +914,7 @@ def test_patch_from_manifest( mock_create_patch.assert_called_with( role=role, package_name="app_pkg", - stage_fqn=f"app_pkg.{stage}", + stage_path_to_artifacts=f"app_pkg.{stage}", version="manifest_version", # cli patch overrides the manifest patch=cli_patch, diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index 6b906272bd..6eee0b1474 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -34,7 +34,7 @@ put_files_on_stage, sync_local_diff_with_stage, ) -from snowflake.cli._plugins.stage.manager import StageManager +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts, StageManager from snowflake.cli._plugins.stage.utils import print_diff_to_console from snowflake.cli.api.exceptions import ( SnowflakeSQLExecutionError, @@ -89,12 +89,13 @@ def stage_contents( ] +# PJ - TODO add tests for following cases with subdir @mock.patch(f"{STAGE_MANAGER}.list_files") def test_empty_stage(mock_list, mock_cursor): mock_list.return_value = mock_cursor(rows=[], columns=STAGE_LS_COLUMNS) with temp_local_dir(FILE_CONTENTS) as local_path: - diff_result = compute_stage_diff(local_path, "a.b.c") + diff_result = compute_stage_diff(local_path, DefaultStagePathParts("a.b.c")) assert len(diff_result.only_on_stage) == 0 assert len(diff_result.different) == 0 assert len(diff_result.identical) == 0 @@ -111,7 +112,7 @@ def test_empty_dir(mock_list, mock_cursor): ) with temp_local_dir({}) as local_path: - diff_result = compute_stage_diff(local_path, "a.b.c") + diff_result = compute_stage_diff(local_path, DefaultStagePathParts("a.b.stage")) assert sorted(diff_result.only_on_stage) == sorted( as_stage_paths(FILE_CONTENTS.keys()) ) @@ -128,7 +129,7 @@ def test_identical_stage(mock_list, mock_cursor): ) with temp_local_dir(FILE_CONTENTS) as local_path: - diff_result = compute_stage_diff(local_path, "a.b.c") + diff_result = compute_stage_diff(local_path, DefaultStagePathParts("a.b.stage")) assert len(diff_result.only_on_stage) == 0 assert len(diff_result.different) == 0 assert sorted(diff_result.identical) == sorted( @@ -147,7 +148,7 @@ def test_new_local_file(mock_list, mock_cursor): with temp_local_dir( {**FILE_CONTENTS, "a/new/README.md": "### I am a new markdown readme"} ) as local_path: - diff_result = compute_stage_diff(local_path, "a.b.c") + diff_result = compute_stage_diff(local_path, DefaultStagePathParts("a.b.stage")) assert len(diff_result.only_on_stage) == 0 assert len(diff_result.different) == 0 assert sorted(diff_result.identical) == sorted( @@ -169,7 +170,7 @@ def test_modified_file(mock_list, mock_cursor): "README.md": "This is a modification to the existing README", } ) as local_path: - diff_result = compute_stage_diff(local_path, "a.b.c") + diff_result = compute_stage_diff(local_path, DefaultStagePathParts("a.b.stage")) assert len(diff_result.only_on_stage) == 0 assert sorted(diff_result.different) == as_stage_paths(["README.md"]) assert sorted(diff_result.identical) == as_stage_paths( @@ -195,7 +196,7 @@ def test_unmodified_file_no_remote_md5sum(mock_list, mock_cursor): ) with temp_local_dir(FILE_CONTENTS) as local_path: - diff_result = compute_stage_diff(local_path, "a.b.c") + diff_result = compute_stage_diff(local_path, DefaultStagePathParts("a.b.stage")) assert len(diff_result.only_on_stage) == 0 assert sorted(diff_result.different) == as_stage_paths(["README.md"]) assert sorted(diff_result.identical) == as_stage_paths( @@ -275,12 +276,14 @@ def test_put_files_on_stage(mock_put, overwrite_param): assert mock_put.mock_calls == expected +# PJ - TODO add test here def test_build_md5_map(mock_cursor): actual = build_md5_map( mock_cursor( rows=stage_contents(FILE_CONTENTS), columns=STAGE_LS_COLUMNS, - ) + ), + "stage", ) expected = { From e95ab861ccf543c586a9fca823c5fd915c931b8a Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 3 Dec 2024 16:43:19 -0500 Subject: [PATCH 17/33] update docstring --- .../cli/_plugins/nativeapp/entities/application_package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 64012a9c16..f17607bda4 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -235,7 +235,9 @@ def action_bundle(self, action_ctx: ActionContext, *args, **kwargs): def action_diff( self, action_ctx: ActionContext, print_to_console: bool, *args, **kwargs ): - + """ + Compute the diff between the local artifacts and the remote ones on the stage. + """ bundle_map = self._bundle() diff = compute_stage_diff( local_root=self.deploy_root, From b663798a232c8e5f2fbfb351a9700cd36d5f1462 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Wed, 4 Dec 2024 09:41:33 -0500 Subject: [PATCH 18/33] add unit tests for DefaultStagePathParts --- tests/stage/test_stage_path.py | 91 ++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/stage/test_stage_path.py b/tests/stage/test_stage_path.py index 9bc4fe42cc..21c5b2885c 100644 --- a/tests/stage/test_stage_path.py +++ b/tests/stage/test_stage_path.py @@ -1,6 +1,7 @@ from __future__ import annotations import pytest +from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli.api.stage_path import StagePath # (path, is_git_repo) @@ -168,3 +169,93 @@ def test_parent_path(path, is_git_repo): def test_root_path(stage_name, path): stage_path = StagePath.from_stage_str(path) assert stage_path.root_path() == StagePath.from_stage_str(f"@{stage_name}") + + +@pytest.mark.parametrize( + "input_path, path, full_path, schema, stage, stage_name", + [ + ( + "db.test_schema.test_stage", + "test_stage", + "db.test_schema.test_stage", + "test_schema", + "db.test_schema.test_stage", + "test_stage", + ), + ( + "db.test_schema.test_stage/subdir", + "test_stage/subdir", + "db.test_schema.test_stage/subdir", + "test_schema", + "db.test_schema.test_stage", + "test_stage", + ), + ( + "db.test_schema.test_stage/nested/dir", + "test_stage/nested/dir", + "db.test_schema.test_stage/nested/dir", + "test_schema", + "db.test_schema.test_stage", + "test_stage", + ), + ( + "test_schema.test_stage/nested/dir", + "test_stage/nested/dir", + "test_schema.test_stage/nested/dir", + "test_schema", + "test_schema.test_stage", + "test_stage", + ), + ( + "test_schema.test_stage/trailing/", + "test_stage/trailing", + "test_schema.test_stage/trailing", + "test_schema", + "test_schema.test_stage", + "test_stage", + ), + ( + "db.test_schema.test_stage/nested/trailing/", + "test_stage/nested/trailing", + "db.test_schema.test_stage/nested/trailing", + "test_schema", + "db.test_schema.test_stage", + "test_stage", + ), + ( + "test_stage/nested/trailing/", + "test_stage/nested/trailing", + "test_stage/nested/trailing", + None, + "test_stage", + "test_stage", + ), + ("test_stage/", "test_stage", "test_stage", None, "test_stage", "test_stage"), + ( + "test_stage/nested/dir", + "test_stage/nested/dir", + "test_stage/nested/dir", + None, + "test_stage", + "test_stage", + ), + ("test_stage", "test_stage", "test_stage", None, "test_stage", "test_stage"), + ( + "test_stage/dir/", + "test_stage/dir", + "test_stage/dir", + None, + "test_stage", + "test_stage", + ), + ], +) +def test_default_stage_path_parts( + input_path, path, full_path, schema, stage, stage_name +): + stage_path_parts = DefaultStagePathParts(input_path) + assert stage_path_parts.full_path == full_path + assert stage_path_parts.schema == schema + assert stage_path_parts.path == path + assert stage_path_parts.stage == stage + assert stage_path_parts.stage_name == stage_name From 08a41106a451848462754ec87d083119fdd352b9 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Wed, 4 Dec 2024 11:28:18 -0500 Subject: [PATCH 19/33] update stage diff command and message wording --- src/snowflake/cli/_plugins/stage/commands.py | 3 ++- src/snowflake/cli/_plugins/stage/diff.py | 6 ++---- src/snowflake/cli/api/entities/utils.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/snowflake/cli/_plugins/stage/commands.py b/src/snowflake/cli/_plugins/stage/commands.py index 9261c7f2da..44c8c9e136 100644 --- a/src/snowflake/cli/_plugins/stage/commands.py +++ b/src/snowflake/cli/_plugins/stage/commands.py @@ -192,7 +192,8 @@ def stage_diff( Diffs a stage with a local folder. """ diff: DiffResult = compute_stage_diff( - local_root=Path(folder_name), stage_fqn=stage_name + local_root=Path(folder_name), + stage_path=StageManager._stage_path_part_factory(stage_name), # noqa: SLF001 ) if get_cli_context().output_format == OutputFormat.JSON: return ObjectResult(diff.to_dict()) diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index 2cd498ca84..83870be20b 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -25,7 +25,7 @@ ) from snowflake.connector.cursor import DictCursor -from .manager import DefaultStagePathParts, StageManager +from .manager import StageManager, StagePathParts from .md5 import UnknownMD5FormatError, file_matches_md5sum log = logging.getLogger(__name__) @@ -122,9 +122,7 @@ def preserve_from_diff( return preserved_diff -def compute_stage_diff( - local_root: Path, stage_path: DefaultStagePathParts -) -> DiffResult: +def compute_stage_diff(local_root: Path, stage_path: StagePathParts) -> DiffResult: """ Diffs the files in the local_root with files in the stage path that is stage_path_parts's full_path. """ diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 888a09a37c..1ada533f7e 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -122,7 +122,7 @@ def sync_deploy_root_with_stage( # Perform a diff operation and display results to the user for informational purposes if print_diff: console.step( - f"Performing a diff between the Snowflake stage {stage_path.path} and your local deploy_root {deploy_root.resolve()} directory." + f"Performing a diff between the Snowflake stage: {stage_path.path} and your local deploy_root: {deploy_root.resolve()}." ) diff: DiffResult = compute_stage_diff( From c7a0a88c5c8ed8a25add150cd4495b45b1f8a1e0 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Wed, 4 Dec 2024 11:37:50 -0500 Subject: [PATCH 20/33] update snapshot for test_deploy --- .../nativeapp/__snapshots__/test_deploy.ambr | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests_integration/nativeapp/__snapshots__/test_deploy.ambr b/tests_integration/nativeapp/__snapshots__/test_deploy.ambr index 2f21ef98b9..5870ccdc61 100644 --- a/tests_integration/nativeapp/__snapshots__/test_deploy.ambr +++ b/tests_integration/nativeapp/__snapshots__/test_deploy.ambr @@ -3,7 +3,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/README.md -> README.md added: app/manifest.yml -> manifest.yml @@ -18,7 +18,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/README.md -> README.md added: app/manifest.yml -> manifest.yml @@ -33,7 +33,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/README.md -> README.md added: app/manifest.yml -> manifest.yml @@ -48,7 +48,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/README.md -> README.md added: app/manifest.yml -> manifest.yml @@ -63,7 +63,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/manifest.yml -> manifest.yml added: app/setup_script.sql -> setup_script.sql @@ -76,7 +76,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/README.md -> README.md added: app/manifest.yml -> manifest.yml @@ -91,7 +91,7 @@ ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Local changes to be deployed: added: app/nested/dir/file.txt -> nested/dir/file.txt Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. @@ -102,7 +102,7 @@ # name: test_nativeapp_deploy_prune[app deploy --no-prune-contains4-not_contains4-napp_init_v2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. The following files exist only on the stage: README.md @@ -116,7 +116,7 @@ # name: test_nativeapp_deploy_prune[app deploy --no-validate-contains2-not_contains2-napp_init_v2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Deleted paths to be removed from your stage: deleted: README.md Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. @@ -127,7 +127,7 @@ # name: test_nativeapp_deploy_prune[app deploy --prune --no-validate-contains0-not_contains0-napp_init_v2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Deleted paths to be removed from your stage: deleted: README.md Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. @@ -138,7 +138,7 @@ # name: test_nativeapp_deploy_prune[ws deploy --entity-id=pkg --no-prune-contains5-not_contains5-napp_init_v2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. The following files exist only on the stage: README.md @@ -152,7 +152,7 @@ # name: test_nativeapp_deploy_prune[ws deploy --entity-id=pkg --no-validate-contains3-not_contains3-napp_init_v2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Deleted paths to be removed from your stage: deleted: README.md Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. @@ -163,7 +163,7 @@ # name: test_nativeapp_deploy_prune[ws deploy --entity-id=pkg --prune --no-validate-contains1-not_contains1-napp_init_v2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. - Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Performing a diff between the Snowflake stage: stage and your local deploy_root: @@DEPLOY_ROOT@@. Deleted paths to be removed from your stage: deleted: README.md Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. From 504f692dc2d317b1a75dca9249d82e14669b46c2 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Wed, 4 Dec 2024 16:57:44 -0500 Subject: [PATCH 21/33] add integration tests --- .../templating/test_templates_processor.py | 13 ++ tests/nativeapp/factories.py | 1 + .../nativeapp/__snapshots__/test_deploy.ambr | 66 ++++++++ tests_integration/nativeapp/test_bundle.py | 47 ++++- tests_integration/nativeapp/test_deploy.py | 160 ++++++++++++++++++ tests_integration/nativeapp/test_validate.py | 60 +++++++ tests_integration/nativeapp/test_version.py | 70 +++++++- .../napp_stage_subdirs/app/v1/README.md | 3 + .../napp_stage_subdirs/app/v1/manifest.yml | 11 ++ .../app/v1/setup_script.sql | 5 + .../napp_stage_subdirs/app/v2/README.md | 3 + .../napp_stage_subdirs/app/v2/manifest.yml | 11 ++ .../app/v2/setup_script.sql | 5 + .../projects/napp_stage_subdirs/snowflake.yml | 31 ++++ .../app/v1/README.md | 3 + .../app/v1/manifest.yml | 11 ++ .../app/v1/module-echo-v1/echo-v1.py | 13 ++ .../app/v1/setup_script.sql | 5 + .../app/v2/README.md | 3 + .../app/v2/manifest.yml | 11 ++ .../app/v2/module-echo-v2/echo-v2.py | 13 ++ .../app/v2/setup_script.sql | 5 + .../snowflake.yml | 38 +++++ 23 files changed, 582 insertions(+), 6 deletions(-) create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml diff --git a/tests/nativeapp/codegen/templating/test_templates_processor.py b/tests/nativeapp/codegen/templating/test_templates_processor.py index 21c7160d06..d1b3488fa4 100644 --- a/tests/nativeapp/codegen/templating/test_templates_processor.py +++ b/tests/nativeapp/codegen/templating/test_templates_processor.py @@ -91,6 +91,19 @@ def test_templates_processor_valid_files_no_templates(): assert bundle_result.output_files[0].read_text() == file_contents[0] +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {}) +def test_templates_processor_cant_read_file(): + file_names = ["test_file.txt"] + file_contents = ["This is a test file\n with some content"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + templates_processor.process(bundle_result.artifact_to_process, None) + + assert bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0] + + @mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"env": {"TEST_VAR": "test_value"}}}) def test_one_file_with_template_and_one_without(): file_names = ["test_file.txt", "test_file_with_template.txt"] diff --git a/tests/nativeapp/factories.py b/tests/nativeapp/factories.py index d0c67f0601..ee4f64874b 100644 --- a/tests/nativeapp/factories.py +++ b/tests/nativeapp/factories.py @@ -132,6 +132,7 @@ class ApplicationPackageEntityModelFactory(EntityModelBaseFactory): artifacts = factory.List( ["setup.sql", "README.md", "manifest.yml"], list_factory=ArtifactFactory ) + stage_subdirectory = None class ApplicationEntityModelFactory(EntityModelBaseFactory): diff --git a/tests_integration/nativeapp/__snapshots__/test_deploy.ambr b/tests_integration/nativeapp/__snapshots__/test_deploy.ambr index 5870ccdc61..b801a4306a 100644 --- a/tests_integration/nativeapp/__snapshots__/test_deploy.ambr +++ b/tests_integration/nativeapp/__snapshots__/test_deploy.ambr @@ -72,6 +72,21 @@ ''' # --- +# name: test_nativeapp_deploy_files_w_stage_subdir + ''' + Creating new application package stage_w_subdirs_pkg_@@USER@@ in account. + Checking if stage stage_w_subdirs_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage: stage/v2 and your local deploy_root: @@DEPLOY_ROOT@@/v2. + Local changes to be deployed: + added: app/v2/README.md -> README.md + added: app/v2/manifest.yml -> manifest.yml + added: app/v2/setup_script.sql -> setup_script.sql + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@/v2 directory. + Validating Snowflake Native App setup script. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- # name: test_nativeapp_deploy_looks_for_prefix_matches[app deploy-napp_deploy_prefix_matches_v2] ''' Creating new application package myapp_pkg_@@USER@@ in account. @@ -171,3 +186,54 @@ ''' # --- +# name: test_nativeapp_deploy_prune_w_stage_subdir[app deploy --package-entity-id=pkg_v1 --no-prune-contains2-not_contains2] + ''' + Checking if stage stage_w_subdirs_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage: stage/v1 and your local deploy_root: @@DEPLOY_ROOT@@/v1. + The following files exist only on the stage: + README.md + + Use the --prune flag to delete them from the stage. + Your stage is up-to-date with your local deploy root. + Validating Snowflake Native App setup script. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_prune_w_stage_subdir[app deploy --package-entity-id=pkg_v1 --no-validate-contains1-not_contains1] + ''' + Checking if stage stage_w_subdirs_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage: stage/v1 and your local deploy_root: @@DEPLOY_ROOT@@/v1. + Deleted paths to be removed from your stage: + deleted: README.md + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@/v1 directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_prune_w_stage_subdir[app deploy --package-entity-id=pkg_v1 --prune --no-validate-contains0-not_contains0] + ''' + Checking if stage stage_w_subdirs_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage: stage/v1 and your local deploy_root: @@DEPLOY_ROOT@@/v1. + Deleted paths to be removed from your stage: + deleted: README.md + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@/v1 directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_w_stage_subdir + ''' + Creating new application package stage_w_subdirs_pkg_@@USER@@ in account. + Checking if stage stage_w_subdirs_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage: stage/v1 and your local deploy_root: @@DEPLOY_ROOT@@/v1. + Local changes to be deployed: + added: app/v1/README.md -> README.md + added: app/v1/manifest.yml -> manifest.yml + added: app/v1/setup_script.sql -> setup_script.sql + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@/v1 directory. + Validating Snowflake Native App setup script. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- diff --git a/tests_integration/nativeapp/test_bundle.py b/tests_integration/nativeapp/test_bundle.py index 654f874574..5c5eb933ba 100644 --- a/tests_integration/nativeapp/test_bundle.py +++ b/tests_integration/nativeapp/test_bundle.py @@ -50,7 +50,9 @@ def template_setup_all(runner, nativeapp_project_directory, request): ) -def _template_setup(runner, nativeapp_project_directory, command, test_project): +def _template_setup( + runner, nativeapp_project_directory, command, test_project, deploy_root_subdir=None +): """ Sets up a project directory and runs the bundle command on the application package. Returns (project_root, execute_bundle_command, test_project) @@ -69,7 +71,7 @@ def _template_setup(runner, nativeapp_project_directory, command, test_project): # The newly created deploy_root is explicitly deleted here, as bundle should take care of it. - deploy_root = Path(project_root, "output", "deploy") + deploy_root = Path(project_root, "output", "deploy", deploy_root_subdir or "") assert Path(deploy_root, "manifest.yml").is_file() assert Path(deploy_root, "setup_script.sql").is_file() assert Path(deploy_root, "README.md").is_file() @@ -318,3 +320,44 @@ def test_nativeapp_bundle_deletes_existing_deploy_root(template_setup): result = execute_bundle_command() assert result.exit_code == 0 assert not existing_deploy_root_dest.exists() + + +@pytest.mark.integration +def test_nativeapp_can_bundle_with_subdirs(runner, nativeapp_project_directory): + command = "app bundle --package-entity-id=pkg_v1" + subdir = "v1" + with nativeapp_project_directory("napp_stage_subdirs") as project_root: + result = runner.invoke_json(split(command)) + assert result.exit_code == 0 + + deploy_root = Path(project_root, "output", "deploy", subdir) + assert Path(deploy_root, "manifest.yml").is_file() + assert Path(deploy_root, "setup_script.sql").is_file() + assert Path(deploy_root, "README.md").is_file() + + +@pytest.mark.integration +def test_nativeapp_bundle_subdirs_dont_overwrite(runner, nativeapp_project_directory): + + with nativeapp_project_directory("napp_stage_subdirs_w_snowpark") as project_root: + result_1 = runner.invoke_json(split("app bundle --package-entity-id=pkg_v1")) + assert result_1.exit_code == 0 + + result_2 = runner.invoke_json(split("app bundle --package-entity-id=pkg_v2")) + assert result_2.exit_code == 0 + + for subdir in ["v1", "v2"]: + deploy_root = Path(project_root, "output", "deploy", subdir) + assert Path(deploy_root, "manifest.yml").is_file() + assert Path(deploy_root, "setup_script.sql").is_file() + assert Path(deploy_root, "README.md").is_file() + assert Path(deploy_root, f"module-echo-{subdir}").is_dir() + assert Path( + deploy_root, f"module-echo-{subdir}", f"echo-{subdir}.py" + ).is_file() + # With snowpark annotation processor + assert Path(deploy_root, "__generated").is_dir() + assert Path( + deploy_root, + f"__generated/snowpark/module-echo-{subdir}/echo-{subdir}.sql", + ).is_file() diff --git a/tests_integration/nativeapp/test_deploy.py b/tests_integration/nativeapp/test_deploy.py index f07710f571..be29b3e10e 100644 --- a/tests_integration/nativeapp/test_deploy.py +++ b/tests_integration/nativeapp/test_deploy.py @@ -106,6 +106,63 @@ def test_nativeapp_deploy( assert "Successfully uploaded chunk 0 of file" not in result.output +@pytest.mark.integration +def test_nativeapp_deploy_w_stage_subdir( + nativeapp_project_directory, + runner, + snowflake_session, + default_username, + resource_suffix, + sanitize_deploy_output, + snapshot, + print_paths_as_posix, +): + project_name = "stage_w_subdirs_pkg" + with nativeapp_project_directory("napp_stage_subdirs"): + result = runner.invoke_with_connection( + split("app deploy --package-entity-id=pkg_v1") + ) + assert result.exit_code == 0 + assert "Validating Snowflake Native App setup script." in result.output + assert sanitize_deploy_output(result.output) == snapshot + + # package exist + package_name = f"{project_name}_{default_username}{resource_suffix}".upper() + app_name = f"{project_name}_{default_username}{resource_suffix}".upper() + assert contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"show application packages like '{package_name}'", + ) + ), + dict(name=package_name), + ) + + # manifest file exists + stage_name = "app_src.stage/v1" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"] + ) + assert contains_row_with(stage_files.json, {"name": "stage/v1/manifest.yml"}) + + # app does not exist + assert not_contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"show applications like '{app_name}'", + ) + ), + dict(name=app_name), + ) + + # re-deploying should be a no-op; make sure we don't issue any PUT commands + result = runner.invoke_with_connection( + [*split("app deploy --package-entity-id=pkg_v1"), "--debug"] + ) + assert result.exit_code == 0 + assert "Successfully uploaded chunk 0 of file" not in result.output + + @pytest.mark.integration @pytest.mark.parametrize( "command,contains,not_contains,test_project", @@ -189,6 +246,70 @@ def test_nativeapp_deploy_prune( assert not_contains_row_with(stage_files.json, {"name": name}) +@pytest.mark.integration +@pytest.mark.parametrize( + "command,contains,not_contains", + [ + # deploy --prune removes remote-only files + [ + "app deploy --package-entity-id=pkg_v1 --prune --no-validate", + ["stage/v1/manifest.yml"], + ["stage/v1/README.md"], + ], + # deploy removes remote-only files (--prune is the default value) + [ + "app deploy --package-entity-id=pkg_v1 --no-validate", + ["stage/v1/manifest.yml"], + ["stage/v1/README.md"], + ], + # deploy --no-prune does not delete remote-only files + [ + "app deploy --package-entity-id=pkg_v1 --no-prune", + ["stage/v1/README.md"], + [], + ], + ], +) +def test_nativeapp_deploy_prune_w_stage_subdir( + command, + contains, + not_contains, + nativeapp_project_directory, + runner, + snapshot, + print_paths_as_posix, + default_username, + resource_suffix, + sanitize_deploy_output, +): + test_project = "napp_stage_subdirs" + project_name = "stage_w_subdirs_pkg" + with nativeapp_project_directory(test_project): + result = runner.invoke_with_connection_json( + ["app", "deploy", "--package-entity-id=pkg_v1"] + ) + assert result.exit_code == 0 + + # delete a file locally + os.remove(os.path.join("app", "v1", "README.md")) + + # deploy + result = runner.invoke_with_connection(split(command)) + assert result.exit_code == 0 + assert sanitize_deploy_output(result.output) == snapshot + + # verify the file does not exist on the stage + package_name = f"{project_name}_{default_username}{resource_suffix}".upper() + stage_name = "app_src.stage/v1" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"] + ) + for name in contains: + assert contains_row_with(stage_files.json, {"name": name}) + for name in not_contains: + assert not_contains_row_with(stage_files.json, {"name": name}) + + # Tests a simple flow of executing "snow app deploy [files]", verifying that only the specified files are synced to the stage @pytest.mark.integration @pytest.mark.parametrize( @@ -232,6 +353,45 @@ def test_nativeapp_deploy_files( assert not_contains_row_with(stage_files.json, {"name": "stage/README.md"}) +@pytest.mark.integration +def test_nativeapp_deploy_files_w_stage_subdir( + nativeapp_project_directory, + runner, + snapshot, + print_paths_as_posix, + default_username, + resource_suffix, + sanitize_deploy_output, +): + project_name = "stage_w_subdirs_pkg" + with nativeapp_project_directory("napp_stage_subdirs"): + # sync only two specific files to stage + touch("app/v2/file.txt") + result = runner.invoke_with_connection( + [ + *split("app deploy --package-entity-id=pkg_v2"), + "app/v2/manifest.yml", + "app/v2/setup_script.sql", + "app/v2/README.md", + ] + ) + assert result.exit_code == 0 + assert sanitize_deploy_output(result.output) == snapshot + + # manifest and script files exist, readme doesn't exist + package_name = f"{project_name}_{default_username}{resource_suffix}".upper() + stage_name = "app_src.stage/v2" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"] + ) + assert contains_row_with(stage_files.json, {"name": "stage/v2/manifest.yml"}) + assert contains_row_with( + stage_files.json, {"name": "stage/v2/setup_script.sql"} + ) + assert contains_row_with(stage_files.json, {"name": "stage/v2/README.md"}) + assert not_contains_row_with(stage_files.json, {"name": "stage/v2/file.txt"}) + + # Tests that files inside of a symlinked directory are deployed @pytest.mark.integration @pytest.mark.parametrize( diff --git a/tests_integration/nativeapp/test_validate.py b/tests_integration/nativeapp/test_validate.py index c435f069a9..741dc31d6e 100644 --- a/tests_integration/nativeapp/test_validate.py +++ b/tests_integration/nativeapp/test_validate.py @@ -79,6 +79,32 @@ def test_nativeapp_validate_v2(command, nativeapp_teardown, runner, temp_dir): assert "Native App validation succeeded." in result.output +@pytest.mark.integration +def test_nativeapp_validate_subdirs(nativeapp_teardown, runner, temp_dir): + ProjectV2Factory( + pdf__entities=dict( + pkg=ApplicationPackageEntityModelFactory( + identifier="myapp_pkg", + stage_subdirectory="v1", + ), + app=ApplicationEntityModelFactory( + identifier="myapp", + fromm__target="pkg", + ), + ), + files={ + "setup.sql": "CREATE OR ALTER VERSIONED SCHEMA core;", + "README.md": "\n", + "manifest.yml": "\n", + }, + ) + with nativeapp_teardown(project_dir=Path(temp_dir)): + # validate the app's setup script + result = runner.invoke_with_connection(split("app validate")) + assert result.exit_code == 0, result.output + assert "Native App validation succeeded." in result.output + + @pytest.mark.integration def test_nativeapp_validate_failing(nativeapp_teardown, runner, temp_dir): ProjectV2Factory( @@ -112,6 +138,40 @@ def test_nativeapp_validate_failing(nativeapp_teardown, runner, temp_dir): assert "syntax error" in result.output +@pytest.mark.integration +def test_nativeapp_validate_failing_w_subdir(nativeapp_teardown, runner, temp_dir): + ProjectV2Factory( + pdf__entities=dict( + pkg=ApplicationPackageEntityModelFactory( + identifier="myapp_pkg", + stage_subdirectory="v1", + ), + app=ApplicationEntityModelFactory( + identifier="myapp", + fromm__target="pkg", + ), + ), + files={ + # Create invalid SQL file + "setup.sql": dedent( + """\ + CREATE OR ALTER VERSIONED SCHEMA core; + Lorem ipsum dolor sit amet + """ + ), + "README.md": "\n", + "manifest.yml": "\n", + }, + ) + with nativeapp_teardown(project_dir=Path(temp_dir)): + # validate the app's setup script, this will fail + # because we include an empty file + result = runner.invoke_with_connection(["app", "validate"]) + assert result.exit_code == 1, result.output + assert "Snowflake Native App setup script failed validation." in result.output + assert "syntax error" in result.output + + @pytest.mark.integration def test_nativeapp_validate_with_post_deploy_hooks( nativeapp_teardown, runner, temp_dir diff --git a/tests_integration/nativeapp/test_version.py b/tests_integration/nativeapp/test_version.py index ce9706afba..8fa664ffa8 100644 --- a/tests_integration/nativeapp/test_version.py +++ b/tests_integration/nativeapp/test_version.py @@ -68,16 +68,36 @@ def temporary_role(request, snowflake_session, resource_suffix): # Tests a simple flow of an existing project, executing snow app version create, drop and teardown, all with distribution=internal @pytest.mark.integration @pytest.mark.parametrize( - "create_command,list_command,drop_command,test_project", + "project_name,create_command,list_command,drop_command,test_project", [ - ["app version create", "app version list", "app version drop", "napp_init_v1"], - ["app version create", "app version list", "app version drop", "napp_init_v2"], [ + "myapp", + "app version create", + "app version list", + "app version drop", + "napp_init_v1", + ], + [ + "myapp", + "app version create", + "app version list", + "app version drop", + "napp_init_v2", + ], + [ + "myapp", "ws version create --entity-id=pkg", "ws version list --entity-id=pkg", "ws version drop --entity-id=pkg", "napp_init_v2", ], + [ + "stage_w_subdirs", + "app version create --package-entity-id=pkg_v1", + "app version list --package-entity-id=pkg_v1", + "app version drop --package-entity-id=pkg_v1", + "napp_stage_subdirs", + ], ], ) def test_nativeapp_version_create_and_drop( @@ -86,12 +106,12 @@ def test_nativeapp_version_create_and_drop( default_username, resource_suffix, nativeapp_project_directory, + project_name, create_command, list_command, drop_command, test_project, ): - project_name = "myapp" with nativeapp_project_directory(test_project): result_create = runner.invoke_with_connection_json( [*split(create_command), "v1", "--force", "--skip-git-check"] @@ -170,6 +190,48 @@ def test_nativeapp_upgrade( runner.invoke_with_connection_json([*split(drop_command), "v1", "--force"]) +@pytest.mark.integration +def test_nativeapp_upgrade_w_subdirs( + runner, + snowflake_session, + default_username, + resource_suffix, + nativeapp_project_directory, +): + project_name = "stage_w_subdirs" + create_command = ( + "app version create v1 --package-entity-id=pkg_v1 --force --skip-git-check" + ) + list_command = "app version list --package-entity-id=pkg_v1" + drop_command = "app version drop --package-entity-id=pkg_v1" + with nativeapp_project_directory("napp_stage_subdirs"): + runner.invoke_with_connection_json(["app", "run", "--app-entity-id=app_v1"]) + runner.invoke_with_connection_json(split(create_command)) + + # package exist + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() + app_name = f"{project_name}_app_v1_{default_username}{resource_suffix}".upper() + # app package contains version v1 + expect = snowflake_session.execute_string( + f"show versions in application package {package_name}" + ) + actual = runner.invoke_with_connection_json(split(list_command)) + assert actual.json == row_from_snowflake_session(expect) + + runner.invoke_with_connection_json( + ["app", "run", "--app-entity-id=app_v1", "--version", "v1", "--force"] + ) + + expect = row_from_snowflake_session( + snowflake_session.execute_string(f"desc application {app_name}") + ) + assert contains_row_with(expect, {"property": "name", "value": app_name}) + assert contains_row_with(expect, {"property": "version", "value": "V1"}) + assert contains_row_with(expect, {"property": "patch", "value": "0"}) + + runner.invoke_with_connection_json([*split(drop_command), "v1", "--force"]) + + # Make sure we can create 3+ patches on the same version @pytest.mark.integration @pytest.mark.parametrize( diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md new file mode 100644 index 0000000000..3107d92de2 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md @@ -0,0 +1,3 @@ +# README + +This is the <% ctx.pkg_v1.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml new file mode 100644 index 0000000000..3fa13420db --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml @@ -0,0 +1,11 @@ +# This is the v2 version of the napp_init_v1 project + +manifest_version: 1 + +version: + name: v1 + +artifacts: + setup_script: setup_script.sql + readme: README.md + extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql new file mode 100644 index 0000000000..18017841cc --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql @@ -0,0 +1,5 @@ + +CREATE APPLICATION ROLE IF NOT EXISTS app_public; + +CREATE OR ALTER VERSIONED SCHEMA core; +GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md new file mode 100644 index 0000000000..b6162eae29 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md @@ -0,0 +1,3 @@ +# README + +This is the <% ctx.pkg_v2.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml new file mode 100644 index 0000000000..ee1f6c99c5 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml @@ -0,0 +1,11 @@ +# This is the v2 version of the napp_init_v1 project + +manifest_version: 1 + +version: + name: v2 + +artifacts: + setup_script: setup_script.sql + readme: README.md + extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql new file mode 100644 index 0000000000..18017841cc --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql @@ -0,0 +1,5 @@ + +CREATE APPLICATION ROLE IF NOT EXISTS app_public; + +CREATE OR ALTER VERSIONED SCHEMA core; +GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml b/tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml new file mode 100644 index 0000000000..c93cab9704 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml @@ -0,0 +1,31 @@ +definition_version: 2 +entities: + pkg_v1: + type: application package + identifier: <% fn.concat_ids('stage_w_subdirs_pkg_', ctx.env.USER) %> + stage_subdirectory: v1 + manifest: "" + artifacts: + - src: app/v1/* + dest: ./ + + pkg_v2: + type: application package + identifier: <% fn.concat_ids('stage_w_subdirs_pkg_', ctx.env.USER) %> + stage_subdirectory: v2 + manifest: "" + artifacts: + - src: app/v2/* + dest: ./ + + app_v1: + type: application + from: + target: pkg_v1 + identifier: <% fn.concat_ids('stage_w_subdirs_app_v1_', ctx.env.USER) %> + + app_v2: + type: application + from: + target: pkg_v2 + identifier: <% fn.concat_ids('stage_w_subdirs_app_v2_', ctx.env.USER) %> diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md new file mode 100644 index 0000000000..3107d92de2 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md @@ -0,0 +1,3 @@ +# README + +This is the <% ctx.pkg_v1.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml new file mode 100644 index 0000000000..3fa13420db --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml @@ -0,0 +1,11 @@ +# This is the v2 version of the napp_init_v1 project + +manifest_version: 1 + +version: + name: v1 + +artifacts: + setup_script: setup_script.sql + readme: README.md + extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py new file mode 100644 index 0000000000..335b485dd3 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py @@ -0,0 +1,13 @@ +# This is where you can create python functions, which can further +# be used to create Snowpark UDFs and Stored Procedures in your setup_script.sql file. + +from snowflake.snowpark.functions import udf + +# UDF example: +# decorated example +@udf( + name="echo_fn", + native_app_params={"schema": "core", "application_roles": ["app_public"]}, +) +def echo_fn(data: str) -> str: + return "echo_fn, v1 implementation: " + data diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql new file mode 100644 index 0000000000..18017841cc --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql @@ -0,0 +1,5 @@ + +CREATE APPLICATION ROLE IF NOT EXISTS app_public; + +CREATE OR ALTER VERSIONED SCHEMA core; +GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md new file mode 100644 index 0000000000..b6162eae29 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md @@ -0,0 +1,3 @@ +# README + +This is the <% ctx.pkg_v2.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml new file mode 100644 index 0000000000..ee1f6c99c5 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml @@ -0,0 +1,11 @@ +# This is the v2 version of the napp_init_v1 project + +manifest_version: 1 + +version: + name: v2 + +artifacts: + setup_script: setup_script.sql + readme: README.md + extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py new file mode 100644 index 0000000000..6ae32cf48a --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py @@ -0,0 +1,13 @@ +# This is where you can create python functions, which can further +# be used to create Snowpark UDFs and Stored Procedures in your setup_script.sql file. + +from snowflake.snowpark.functions import udf + +# UDF example: +# decorated example +@udf( + name="echo_fn", + native_app_params={"schema": "core", "application_roles": ["app_public"]}, +) +def echo_fn(data: str) -> str: + return "echo_fn, v2 implementation: " + data diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql new file mode 100644 index 0000000000..18017841cc --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql @@ -0,0 +1,5 @@ + +CREATE APPLICATION ROLE IF NOT EXISTS app_public; + +CREATE OR ALTER VERSIONED SCHEMA core; +GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml new file mode 100644 index 0000000000..1f18f52062 --- /dev/null +++ b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml @@ -0,0 +1,38 @@ +definition_version: 2 +entities: + pkg_v1: + type: application package + identifier: <% fn.concat_ids('stage_w_subdirs_pkg', ctx.env.suffix) %> + stage_subdirectory: v1 + manifest: "" + artifacts: + - src: app/v1/* + dest: ./ + processors: + - snowpark + + pkg_v2: + type: application package + identifier: <% fn.concat_ids('stage_w_subdirs_pkg', ctx.env.suffix) %> + stage_subdirectory: v2 + manifest: "" + artifacts: + - src: app/v2/* + dest: ./ + processors: + - snowpark + + app_v1: + type: application + from: + target: pkg_v1 + identifier: <% fn.concat_ids('stage_w_subdirs_app_v1', ctx.env.suffix) %> + + app_v2: + type: application + from: + target: pkg_v2 + identifier: <% fn.concat_ids('stage_w_subdirs_app_v2', ctx.env.suffix) %> + +env: + suffix: <% fn.concat_ids('_', fn.sanitize_id(fn.get_username('unknown_user')) | lower) %> From b81670c5874857e49aff4d1ef0de0e29ffa5e4e4 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Thu, 5 Dec 2024 11:17:12 -0500 Subject: [PATCH 22/33] add unit tests --- .../templating/test_templates_processor.py | 13 -- tests/nativeapp/factories.py | 2 +- tests/nativeapp/fixtures.py | 12 +- .../test_application_package_entity.py | 180 +++++++++++++++++- tests/nativeapp/test_event_sharing.py | 105 +++++++++- tests/nativeapp/test_manager.py | 64 +++++++ 6 files changed, 351 insertions(+), 25 deletions(-) diff --git a/tests/nativeapp/codegen/templating/test_templates_processor.py b/tests/nativeapp/codegen/templating/test_templates_processor.py index d1b3488fa4..21c7160d06 100644 --- a/tests/nativeapp/codegen/templating/test_templates_processor.py +++ b/tests/nativeapp/codegen/templating/test_templates_processor.py @@ -91,19 +91,6 @@ def test_templates_processor_valid_files_no_templates(): assert bundle_result.output_files[0].read_text() == file_contents[0] -@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {}) -def test_templates_processor_cant_read_file(): - file_names = ["test_file.txt"] - file_contents = ["This is a test file\n with some content"] - with TemporaryDirectory() as tmp_dir: - bundle_result = bundle_files(tmp_dir, file_names, file_contents) - templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) - templates_processor.process(bundle_result.artifact_to_process, None) - - assert bundle_result.output_files[0].is_symlink() - assert bundle_result.output_files[0].read_text() == file_contents[0] - - @mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"env": {"TEST_VAR": "test_value"}}}) def test_one_file_with_template_and_one_without(): file_names = ["test_file.txt", "test_file_with_template.txt"] diff --git a/tests/nativeapp/factories.py b/tests/nativeapp/factories.py index ee4f64874b..9e88d085c2 100644 --- a/tests/nativeapp/factories.py +++ b/tests/nativeapp/factories.py @@ -132,7 +132,7 @@ class ApplicationPackageEntityModelFactory(EntityModelBaseFactory): artifacts = factory.List( ["setup.sql", "README.md", "manifest.yml"], list_factory=ArtifactFactory ) - stage_subdirectory = None + stage_subdirectory = "" class ApplicationEntityModelFactory(EntityModelBaseFactory): diff --git a/tests/nativeapp/fixtures.py b/tests/nativeapp/fixtures.py index f137bb9491..74343d2568 100644 --- a/tests/nativeapp/fixtures.py +++ b/tests/nativeapp/fixtures.py @@ -38,14 +38,18 @@ def mock_bundle_map(): @pytest.fixture() -def application_package_entity(workspace_context): - data = ApplicationPackageEntityModelFactory(identifier=factory.Faker("word")) +def application_package_entity(workspace_context, request): + pkg_params = getattr(request, "param", {}) + data = ApplicationPackageEntityModelFactory( + identifier=factory.Faker("word"), **pkg_params + ) model = ApplicationPackageEntityModel(**data) return ApplicationPackageEntity(model, workspace_context) @pytest.fixture() -def application_entity(workspace_context): - data = ApplicationEntityModelFactory(identifier=factory.Faker("word")) +def application_entity(workspace_context, request): + app_params = getattr(request, "param", {}) + data = ApplicationEntityModelFactory(identifier=factory.Faker("word"), **app_params) model = ApplicationEntityModel(**data) return ApplicationEntity(model, workspace_context) diff --git a/tests/nativeapp/test_application_package_entity.py b/tests/nativeapp/test_application_package_entity.py index 96bdbe481e..f3c29eed1d 100644 --- a/tests/nativeapp/test_application_package_entity.py +++ b/tests/nativeapp/test_application_package_entity.py @@ -16,6 +16,7 @@ from pathlib import Path from unittest import mock +import pytest import yaml from snowflake.cli._plugins.connection.util import UIParameter from snowflake.cli._plugins.nativeapp.constants import ( @@ -30,6 +31,11 @@ from snowflake.cli._plugins.workspace.context import ActionContext, WorkspaceContext from snowflake.connector.cursor import DictCursor +from tests.nativeapp.factories import ( + ApplicationEntityModelFactory, + ApplicationPackageEntityModelFactory, + ProjectV2Factory, +) from tests.nativeapp.utils import ( APP_PACKAGE_ENTITY, APPLICATION_PACKAGE_ENTITY_MODULE, @@ -39,10 +45,14 @@ ) -def _get_app_pkg_entity(project_directory): +def _get_app_pkg_entity(project_directory, package_overrides=None): with project_directory("workspaces_simple") as project_root: with Path(project_root / "snowflake.yml").open() as definition_file_path: project_definition = yaml.safe_load(definition_file_path) + # project_definition["entities"]["pkg"]["stage_subdirectory"] = None + project_definition["entities"]["pkg"] = dict( + project_definition["entities"]["pkg"], **(package_overrides or {}) + ) model = ApplicationPackageEntityModel( **project_definition["entities"]["pkg"] ) @@ -63,6 +73,39 @@ def _get_app_pkg_entity(project_directory): ) +def package_with_subdir_factory(): + ProjectV2Factory( + pdf__entities=dict( + pkg=ApplicationPackageEntityModelFactory( + identifier="myapp_pkg", stage_subdirectory="v1" + ), + app=ApplicationEntityModelFactory( + identifier="myapp", + fromm__target="pkg", + ), + ), + files={ + "setup.sql": "SELECT 1;", + "README.md": "Hello!", + "manifest.yml": "\n", + }, + ) + + +def test_bundle_with_subdir(project_directory): + package_with_subdir_factory() + app_pkg, bundle_ctx, mock_console = _get_app_pkg_entity( + project_directory, {"stage_subdirectory": "v1"} + ) + + bundle_result = app_pkg.action_bundle(bundle_ctx) + + deploy_root = bundle_result.deploy_root() + assert (deploy_root / "README.md").exists() + assert (deploy_root / "manifest.yml").exists() + assert (deploy_root / "setup_script.sql").exists() + + def test_bundle(project_directory): app_pkg, bundle_ctx, mock_console = _get_app_pkg_entity(project_directory) @@ -174,6 +217,107 @@ def test_deploy( assert mock_execute.mock_calls == expected +@mock.patch(SQL_EXECUTOR_EXECUTE) +@mock.patch(f"{APP_PACKAGE_ENTITY}.execute_post_deploy_hooks") +@mock.patch(f"{APP_PACKAGE_ENTITY}.validate_setup_script") +@mock.patch(f"{APPLICATION_PACKAGE_ENTITY_MODULE}.sync_deploy_root_with_stage") +@mock.patch(SQL_FACADE_GET_UI_PARAMETER, return_value="ENABLED") +def test_deploy_w_stage_subdir( + mock_get_parameter, + mock_sync, + mock_validate, + mock_execute_post_deploy_hooks, + mock_execute, + project_directory, + mock_cursor, +): + side_effects, expected = mock_execute_helper( + [ + ( + mock_cursor([("old_role",)], []), + mock.call("select current_role()"), + ), + (None, mock.call("use role app_role")), + ( + mock_cursor( + [ + { + "name": "PKG", + "comment": SPECIAL_COMMENT, + "version": LOOSE_FILES_MAGIC_VERSION, + "owner": "app_role", + } + ], + [], + ), + mock.call( + r"show application packages like 'PKG'", + cursor_class=DictCursor, + ), + ), + (None, mock.call("use role old_role")), + ( + mock_cursor([("old_role",)], []), + mock.call("select current_role()"), + ), + (None, mock.call("use role app_role")), + ( + mock_cursor( + [ + ("name", "pkg"), + ["owner", "app_role"], + ["distribution", "internal"], + ], + [], + ), + mock.call("describe application package pkg"), + ), + (None, mock.call("use role old_role")), + ( + mock_cursor([("old_role",)], []), + mock.call("select current_role()"), + ), + (None, mock.call("use role app_role")), + (None, mock.call("use role old_role")), + ] + ) + mock_execute.side_effect = side_effects + + app_pkg, bundle_ctx, mock_console = _get_app_pkg_entity( + project_directory, {"stage_subdirectory": "v1"} + ) + + app_pkg.action_deploy( + bundle_ctx, + prune=False, + recursive=False, + paths=["a/b", "c"], + validate=True, + interactive=False, + force=False, + ) + + project_root = app_pkg._workspace_ctx.project_root # noqa SLF001 + mock_sync.assert_called_once_with( + console=mock_console, + deploy_root=(project_root / Path("output/deploy") / "v1"), + package_name="pkg", + bundle_map=mock.ANY, + role="app_role", + prune=False, + recursive=False, + stage_path=DefaultStagePathParts("pkg.app_src.stage/v1"), + local_paths_to_sync=["a/b", "c"], + print_diff=True, + ) + mock_validate.assert_called_once() + mock_execute_post_deploy_hooks.assert_called_once_with() + mock_get_parameter.assert_called_once_with( + UIParameter.NA_FEATURE_RELEASE_CHANNELS, "ENABLED" + ) + assert mock_execute.mock_calls == expected + + @mock.patch(SQL_EXECUTOR_EXECUTE) def test_version_list( mock_execute, application_package_entity, action_context, mock_cursor @@ -189,7 +333,39 @@ def test_version_list( (None, mock.call(f"use role {pkg_model.meta.role}")), ( mock_cursor([], []), - mock.call(f"show versions in application package {pkg_model.fqn.name}"), + mock.call( + f"show versions in application package {application_package_entity.name}" + ), + ), + (None, mock.call("use role old_role")), + ] + ) + mock_execute.side_effect = side_effects + application_package_entity.action_version_list(action_context) + assert mock_execute.mock_calls == expected + + +@mock.patch(SQL_EXECUTOR_EXECUTE) +@pytest.mark.parametrize( + "application_package_entity", [{"stage_subdirectory": "v1"}], indirect=True +) +def test_version_list_w_subdir( + mock_execute, application_package_entity, action_context, mock_cursor +): + pkg_model = application_package_entity._entity_model # noqa SLF001 + pkg_model.meta.role = "package_role" + side_effects, expected = mock_execute_helper( + [ + ( + mock_cursor([("old_role",)], []), + mock.call("select current_role()"), + ), + (None, mock.call(f"use role {pkg_model.meta.role}")), + ( + mock_cursor([], []), + mock.call( + f"show versions in application package {application_package_entity.name}" + ), ), (None, mock.call("use role old_role")), ] diff --git a/tests/nativeapp/test_event_sharing.py b/tests/nativeapp/test_event_sharing.py index ba522543bd..40effe96f6 100644 --- a/tests/nativeapp/test_event_sharing.py +++ b/tests/nativeapp/test_event_sharing.py @@ -37,7 +37,6 @@ SameAccountInstallMethod, ) from snowflake.cli._plugins.nativeapp.sf_facade_exceptions import UserInputError -from snowflake.cli._plugins.stage.manager import DefaultStagePathParts from snowflake.cli._plugins.workspace.context import ActionContext, WorkspaceContext from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.console.abc import AbstractConsole @@ -146,13 +145,12 @@ def _create_or_upgrade_app( ) app = ApplicationEntity(app_model, ctx) pkg = ApplicationPackageEntity(pkg_model, ctx) - stage_fqn = f"{pkg_model.fqn.name}.{pkg_model.stage}" pkg.action_bundle(action_ctx=ActionContext(get_entity=lambda *args: None)) return app.create_or_upgrade_app( package=pkg, - stage_path=DefaultStagePathParts(stage_fqn), + stage_path=pkg.stage_path, install_method=install_method, policy=policy, interactive=is_interactive, @@ -169,6 +167,7 @@ def _setup_project( manifest_contents=test_manifest_contents, share_mandatory_events=None, optional_shared_events=None, + stage_subdirectory="", ): telemetry = {} if share_mandatory_events is not None: @@ -180,6 +179,7 @@ def _setup_project( app_pkg=ApplicationPackageEntityModelFactory( identifier="app_pkg", meta={"role": app_pkg_role, "warehouse": app_pkg_warehouse}, + stage_subdirectory=stage_subdirectory, ), myapp=ApplicationEntityModelFactory( identifier="myapp", @@ -208,6 +208,7 @@ def _setup_mocks_for_app( is_upgrade=False, events_definitions_in_app=None, error_raised=None, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, ): if is_upgrade: return _setup_mocks_for_upgrade_app( @@ -220,6 +221,7 @@ def _setup_mocks_for_app( is_prod=is_prod, events_definitions_in_app=events_definitions_in_app, error_raised=error_raised, + stage_path_to_artifacts=stage_path_to_artifacts, ) else: return _setup_mocks_for_create_app( @@ -232,6 +234,7 @@ def _setup_mocks_for_app( is_prod=is_prod, events_definitions_in_app=events_definitions_in_app, error_raised=error_raised, + stage_path_to_artifacts=stage_path_to_artifacts, ) @@ -245,6 +248,7 @@ def _setup_mocks_for_create_app( events_definitions_in_app=None, is_prod=False, error_raised=None, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, ): mock_get_existing_app_info.return_value = None @@ -298,7 +302,7 @@ def _setup_mocks_for_create_app( install_method=SameAccountInstallMethod.release_directive() if is_prod else SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=stage_path_to_artifacts, debug_mode=None, should_authorize_event_sharing=expected_authorize_telemetry_flag, role="app_role", @@ -347,6 +351,7 @@ def _setup_mocks_for_upgrade_app( events_definitions_in_app=None, is_prod=False, error_raised=None, + stage_path_to_artifacts=DEFAULT_STAGE_FQN, ): mock_get_existing_app_info_result = { "comment": "GENERATED_BY_SNOWFLAKECLI", @@ -401,7 +406,7 @@ def _setup_mocks_for_upgrade_app( install_method=SameAccountInstallMethod.release_directive() if is_prod else SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + stage_path_to_artifacts=stage_path_to_artifacts, debug_mode=None, should_authorize_event_sharing=expected_authorize_telemetry_flag, role="app_role", @@ -449,6 +454,7 @@ def _setup_mocks_for_upgrade_app( "is_upgrade", [False, True], ) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_event_sharing_disabled_no_change_to_current_behavior( mock_param, mock_conn, @@ -460,6 +466,7 @@ def test_event_sharing_disabled_no_change_to_current_behavior( manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -472,11 +479,15 @@ def test_event_sharing_disabled_no_change_to_current_behavior( mock_get_existing_app_info, is_prod=not install_method.is_dev_mode, is_upgrade=is_upgrade, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -525,6 +536,7 @@ def test_event_sharing_disabled_no_change_to_current_behavior( ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_event_sharing_disabled_but_we_add_event_sharing_flag_in_project_definition_file( mock_param, mock_conn, @@ -536,6 +548,7 @@ def test_event_sharing_disabled_but_we_add_event_sharing_flag_in_project_definit manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -548,12 +561,16 @@ def test_event_sharing_disabled_but_we_add_event_sharing_flag_in_project_definit mock_get_existing_app_info, is_prod=not install_method.is_dev_mode, is_upgrade=is_upgrade, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -606,6 +623,7 @@ def test_event_sharing_disabled_but_we_add_event_sharing_flag_in_project_definit ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_event_sharing_enabled_not_enforced_no_mandatory_events_then_flag_respected( mock_param, mock_conn, @@ -617,6 +635,7 @@ def test_event_sharing_enabled_not_enforced_no_mandatory_events_then_flag_respec manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -631,11 +650,15 @@ def test_event_sharing_enabled_not_enforced_no_mandatory_events_then_flag_respec expected_authorize_telemetry_flag=share_mandatory_events, is_upgrade=is_upgrade, expected_shared_events=[] if share_mandatory_events else None, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -683,6 +706,7 @@ def test_event_sharing_enabled_not_enforced_no_mandatory_events_then_flag_respec ], ) @pytest.mark.parametrize("is_upgrade", [True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_event_sharing_enabled_when_upgrade_flag_matches_existing_app_then_do_not_set_it_explicitly( mock_param, mock_conn, @@ -694,6 +718,7 @@ def test_event_sharing_enabled_when_upgrade_flag_matches_existing_app_then_do_no manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -708,11 +733,15 @@ def test_event_sharing_enabled_when_upgrade_flag_matches_existing_app_then_do_no expected_authorize_telemetry_flag=share_mandatory_events, is_upgrade=is_upgrade, expected_shared_events=[] if share_mandatory_events else None, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, # requested flag from the project definition file + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -758,6 +787,7 @@ def test_event_sharing_enabled_when_upgrade_flag_matches_existing_app_then_do_no ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_event_sharing_enabled_with_mandatory_events_and_explicit_authorization_then_flag_respected( mock_param, mock_conn, @@ -769,6 +799,7 @@ def test_event_sharing_enabled_with_mandatory_events_and_explicit_authorization_ manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -791,11 +822,15 @@ def test_event_sharing_enabled_with_mandatory_events_and_explicit_authorization_ "status": "ENABLED", } ], + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -841,6 +876,7 @@ def test_event_sharing_enabled_with_mandatory_events_and_explicit_authorization_ ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_event_sharing_enabled_with_mandatory_events_but_no_authorization_then_flag_respected_with_warning( mock_param, mock_conn, @@ -852,6 +888,7 @@ def test_event_sharing_enabled_with_mandatory_events_but_no_authorization_then_f manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -874,11 +911,15 @@ def test_event_sharing_enabled_with_mandatory_events_but_no_authorization_then_f "status": "ENABLED", } ], + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -929,6 +970,7 @@ def test_event_sharing_enabled_with_mandatory_events_but_no_authorization_then_f ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_no_mandatory_events_then_use_value_provided_for_authorization( mock_param, mock_conn, @@ -940,6 +982,7 @@ def test_enforced_events_sharing_with_no_mandatory_events_then_use_value_provide manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -954,11 +997,15 @@ def test_enforced_events_sharing_with_no_mandatory_events_then_use_value_provide expected_authorize_telemetry_flag=share_mandatory_events, is_upgrade=is_upgrade, expected_shared_events=[] if share_mandatory_events else None, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -1004,6 +1051,7 @@ def test_enforced_events_sharing_with_no_mandatory_events_then_use_value_provide ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_mandatory_events_and_authorization_provided( mock_param, mock_conn, @@ -1016,6 +1064,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_provide share_mandatory_events, install_method, is_upgrade, + stage_subdir, temp_dir, mock_cursor, ): @@ -1029,10 +1078,14 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_provide expected_authorize_telemetry_flag=share_mandatory_events, is_upgrade=is_upgrade, expected_shared_events=[] if share_mandatory_events else None, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, + stage_subdirectory=stage_subdir, share_mandatory_events=share_mandatory_events, ) mock_console = MagicMock() @@ -1079,6 +1132,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_provide ], ) @pytest.mark.parametrize("is_upgrade", [False]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_mandatory_events_and_authorization_refused_on_create_then_error( mock_param, mock_conn, @@ -1090,6 +1144,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_refused manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1117,10 +1172,14 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_refused ), ProgrammingError(errno=APPLICATION_REQUIRES_TELEMETRY_SHARING), ), + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, + stage_subdirectory=stage_subdir, share_mandatory_events=share_mandatory_events, ) mock_console = MagicMock() @@ -1165,6 +1224,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_refused ], ) @pytest.mark.parametrize("is_upgrade", [True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_mandatory_events_manifest_and_authorization_refused_on_update_then_error( mock_param, mock_conn, @@ -1176,6 +1236,7 @@ def test_enforced_events_sharing_with_mandatory_events_manifest_and_authorizatio manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1203,10 +1264,14 @@ def test_enforced_events_sharing_with_mandatory_events_manifest_and_authorizatio ), ProgrammingError(errno=CANNOT_DISABLE_MANDATORY_TELEMETRY), ), + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, + stage_subdirectory=stage_subdir, share_mandatory_events=share_mandatory_events, ) mock_console = MagicMock() @@ -1250,6 +1315,7 @@ def test_enforced_events_sharing_with_mandatory_events_manifest_and_authorizatio ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_mandatory_events_and_dev_mode_then_default_to_true_with_warning( mock_param, mock_conn, @@ -1261,6 +1327,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_dev_mode_then_default manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1275,10 +1342,14 @@ def test_enforced_events_sharing_with_mandatory_events_and_dev_mode_then_default expected_authorize_telemetry_flag=True, is_upgrade=is_upgrade, expected_shared_events=[], + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, + stage_subdirectory=stage_subdir, share_mandatory_events=share_mandatory_events, ) mock_console = MagicMock() @@ -1327,6 +1398,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_dev_mode_then_default ], ) @pytest.mark.parametrize("is_upgrade", [False]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_specified_on_create_and_prod_mode_then_error( mock_param, mock_conn, @@ -1338,6 +1410,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_spe manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1365,10 +1438,14 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_spe ), ProgrammingError(errno=APPLICATION_REQUIRES_TELEMETRY_SHARING), ), + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, + stage_subdirectory=stage_subdir, share_mandatory_events=share_mandatory_events, ) mock_console = MagicMock() @@ -1412,6 +1489,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_spe ], ) @pytest.mark.parametrize("is_upgrade", [True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_specified_on_update_and_prod_mode_then_no_error( mock_param, mock_conn, @@ -1423,6 +1501,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_spe manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1444,10 +1523,14 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_spe "status": "ENABLED", } ], + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, + stage_subdirectory=stage_subdir, share_mandatory_events=share_mandatory_events, ) mock_console = MagicMock() @@ -1493,6 +1576,7 @@ def test_enforced_events_sharing_with_mandatory_events_and_authorization_not_spe ], ) @pytest.mark.parametrize("is_upgrade", [False]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_shared_events_with_no_enabled_mandatory_events_then_error( mock_param, mock_conn, @@ -1504,6 +1588,7 @@ def test_shared_events_with_no_enabled_mandatory_events_then_error( manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1517,12 +1602,16 @@ def test_shared_events_with_no_enabled_mandatory_events_then_error( is_prod=not install_method.is_dev_mode, expected_authorize_telemetry_flag=share_mandatory_events, is_upgrade=is_upgrade, + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, optional_shared_events=["DEBUG_LOGS"], + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() @@ -1566,6 +1655,7 @@ def test_shared_events_with_no_enabled_mandatory_events_then_error( ], ) @pytest.mark.parametrize("is_upgrade", [False, True]) +@pytest.mark.parametrize("stage_subdir", ["", "v1"]) def test_shared_events_with_authorization_then_success( mock_param, mock_conn, @@ -1577,6 +1667,7 @@ def test_shared_events_with_authorization_then_success( manifest_contents, share_mandatory_events, install_method, + stage_subdir, is_upgrade, temp_dir, mock_cursor, @@ -1606,12 +1697,16 @@ def test_shared_events_with_authorization_then_success( "status": "ENABLED", }, ], + stage_path_to_artifacts=f"{DEFAULT_STAGE_FQN}/{stage_subdir}" + if stage_subdir + else DEFAULT_STAGE_FQN, ) mock_conn.return_value = MockConnectionCtx() _setup_project( manifest_contents=manifest_contents, share_mandatory_events=share_mandatory_events, optional_shared_events=shared_events, + stage_subdirectory=stage_subdir, ) mock_console = MagicMock() diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index 93181ebf5c..c66f6eb51b 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -170,6 +170,70 @@ def test_sync_deploy_root_with_stage( ) +@mock.patch(SQL_FACADE_STAGE_EXISTS) +@mock.patch(SQL_FACADE_CREATE_SCHEMA) +@mock.patch(SQL_FACADE_CREATE_STAGE) +@mock.patch(f"{ENTITIES_UTILS_MODULE}.compute_stage_diff") +@mock.patch(f"{ENTITIES_UTILS_MODULE}.sync_local_diff_with_stage") +@pytest.mark.parametrize("stage_exists", [True, False]) +def test_sync_deploy_root_with_stage_subdir( + mock_local_diff_with_stage, + mock_compute_stage_diff, + mock_create_stage, + mock_create_schema, + mock_stage_exists, + temp_dir, + mock_cursor, + stage_exists, +): + mock_stage_exists.return_value = stage_exists + mock_diff_result = DiffResult(different=[StagePathType("setup.sql")]) + mock_compute_stage_diff.return_value = mock_diff_result + mock_local_diff_with_stage.return_value = None + current_working_directory = os.getcwd() + create_named_file( + file_name="snowflake.yml", + dir_name=current_working_directory, + contents=[mock_snowflake_yml_file_v2], + ) + dm = _get_dm() + # add subdir replace + dm.project_definition.entities["app_pkg"].stage_subdirectory = "v1" + pkg_model: ApplicationPackageEntityModel = dm.project_definition.entities["app_pkg"] + + assert mock_diff_result.has_changes() + mock_bundle_map = mock.Mock(spec=BundleMap) + package_name = pkg_model.fqn.name + stage_fqn = f"{package_name}.{pkg_model.stage}" + stage_full_path = f"{stage_fqn}/v1" + stage_schema = extract_schema(stage_fqn) + sync_deploy_root_with_stage( + console=cc, + deploy_root=dm.project_root / pkg_model.deploy_root, + package_name=package_name, + bundle_map=mock_bundle_map, + role="new_role", + prune=True, + recursive=True, + stage_path=DefaultStagePathParts(stage_full_path), + ) + + mock_stage_exists.assert_called_once_with(stage_fqn) + if not stage_exists: + mock_create_schema.assert_called_once_with(stage_schema, database=package_name) + mock_create_stage.assert_called_once_with(stage_fqn) + mock_compute_stage_diff.assert_called_once_with( + local_root=dm.project_root / pkg_model.deploy_root, + stage_path=DefaultStagePathParts(stage_full_path), + ) + mock_local_diff_with_stage.assert_called_once_with( + role="new_role", + deploy_root_path=dm.project_root / pkg_model.deploy_root, + diff_result=mock_diff_result, + stage_full_path=stage_full_path, + ) + + @mock.patch(SQL_FACADE_STAGE_EXISTS) @mock.patch(SQL_EXECUTOR_EXECUTE) @mock.patch(f"{ENTITIES_UTILS_MODULE}.sync_local_diff_with_stage") From 889c77e2bc3ae9efbdf4d51e0f745c5d9e15e860 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Thu, 5 Dec 2024 11:33:38 -0500 Subject: [PATCH 23/33] update release notes --- RELEASE-NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index b5f3ccff40..d316575ac8 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -19,7 +19,7 @@ ## Deprecations ## New additions - +* Added an optional `stage_subdirectory` field to `application package` entity. This field specifies a subdirectory on the stage where the artifacts are to be found. The `manifest.yml` is at the root of this subdirectory. ## Fixes and improvements * Fixed crashes with older x86_64 Intel CPUs. From 44263ce4f683eca1877e06d6acdb6a583932da42 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Thu, 5 Dec 2024 13:29:40 -0500 Subject: [PATCH 24/33] remove template processor change --- .../codegen/templates/templates_processor.py | 69 +++++++++---------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index a16e1dc933..9e38eecc2c 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -58,45 +58,38 @@ def expand_templates_in_file( if src.is_dir(): return - src_file_name = src.relative_to(self._bundle_ctx.project_root) - - try: - with self.edit_file(dest) as file: - if not has_client_side_templates(file.contents) and not ( - _is_sql_file(dest) and has_sql_templates(file.contents) - ): - return - cc.step(f"Expanding templates in {src_file_name}") - with cc.indented(): - try: - jinja_env = ( - choose_sql_jinja_env_based_on_template_syntax( - file.contents, reference_name=src_file_name - ) - if _is_sql_file(dest) - else get_client_side_jinja_env() + with self.edit_file(dest) as file: + if not has_client_side_templates(file.contents) and not ( + _is_sql_file(dest) and has_sql_templates(file.contents) + ): + return + + src_file_name = src.relative_to(self._bundle_ctx.project_root) + cc.step(f"Expanding templates in {src_file_name}") + with cc.indented(): + try: + jinja_env = ( + choose_sql_jinja_env_based_on_template_syntax( + file.contents, reference_name=src_file_name ) - expanded_template = jinja_env.from_string(file.contents).render( - template_context or get_cli_context().template_context - ) - - # For now, we are printing the source file path in the error message - # instead of the destination file path to make it easier for the user - # to identify the file that has the error, and edit the correct file. - except jinja2.TemplateSyntaxError as e: - raise InvalidTemplateInFileError( - src_file_name, e, e.lineno - ) from e - - except jinja2.UndefinedError as e: - raise InvalidTemplateInFileError(src_file_name, e) from e - - if expanded_template != file.contents: - file.edited_contents = expanded_template - except UnicodeDecodeError as err: - cc.warning( - f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file." - ) + if _is_sql_file(dest) + else get_client_side_jinja_env() + ) + expanded_template = jinja_env.from_string(file.contents).render( + template_context or get_cli_context().template_context + ) + + # For now, we are printing the source file path in the error message + # instead of the destination file path to make it easier for the user + # to identify the file that has the error, and edit the correct file. + except jinja2.TemplateSyntaxError as e: + raise InvalidTemplateInFileError(src_file_name, e, e.lineno) from e + + except jinja2.UndefinedError as e: + raise InvalidTemplateInFileError(src_file_name, e) from e + + if expanded_template != file.contents: + file.edited_contents = expanded_template @span("templates_processor") def process( From 70e3d382da8be70c3d9faa09a0796c9b931ae785 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Fri, 6 Dec 2024 17:01:34 -0500 Subject: [PATCH 25/33] make stage paths work for quoted ids --- src/snowflake/cli/_plugins/stage/manager.py | 16 +- tests/stage/test_stage_path.py | 209 ++++++++++++++++++++ 2 files changed, 220 insertions(+), 5 deletions(-) diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index 71b4e18cdd..e7b0625595 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -41,7 +41,7 @@ from snowflake.cli.api.console import cli_console from snowflake.cli.api.constants import PYTHON_3_12 from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.project.util import extract_schema, to_string_literal +from snowflake.cli.api.project.util import VALID_IDENTIFIER_REGEX, to_string_literal from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.cli.api.stage_path import StagePath @@ -65,6 +65,7 @@ # Replace magic numbers with constants OMIT_FIRST = slice(1, None) +STAGE_PATH_REGEX = rf"(?P@)?(?:(?P{VALID_IDENTIFIER_REGEX})\.)?(?:(?P{VALID_IDENTIFIER_REGEX})\.)?(?P{VALID_IDENTIFIER_REGEX})/?(?P([^/]*/?)*)?" @dataclass @@ -132,9 +133,14 @@ class DefaultStagePathParts(StagePathParts): """ def __init__(self, stage_path: str): - self.directory = self.get_directory(stage_path) - self.stage = StageManager.get_stage_from_path(stage_path) - stage_name = self.stage.split(".")[-1] + match = re.fullmatch(STAGE_PATH_REGEX, stage_path) + if match is None: + raise ClickException("Invalid stage path") + self.directory = match.group("directory") + self._schema = match.group("second_qualifier") or match.group("first_qualifier") + self.stage = stage_path.removesuffix(self.directory).rstrip("/") + + stage_name = FQN.from_stage(self.stage).name stage_name = ( stage_name[OMIT_FIRST] if stage_name.startswith("@") else stage_name ) @@ -151,7 +157,7 @@ def full_path(self) -> str: @property def schema(self) -> str | None: - return extract_schema(self.stage) + return self._schema def replace_stage_prefix(self, file_path: str) -> str: stage = Path(self.stage).parts[0] diff --git a/tests/stage/test_stage_path.py b/tests/stage/test_stage_path.py index 21c5b2885c..a38cec6276 100644 --- a/tests/stage/test_stage_path.py +++ b/tests/stage/test_stage_path.py @@ -248,6 +248,215 @@ def test_root_path(stage_name, path): "test_stage", "test_stage", ), + ( + "test_schema.test_stage/nested/dir/file.name", + "test_stage/nested/dir/file.name", + "test_schema.test_stage/nested/dir/file.name", + "test_schema", + "test_schema.test_stage", + "test_stage", + ), + ( + 'db.schema."stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + 'db.schema."stage.sub/dir"/v1', + "schema", + 'db.schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@db.schema."stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + '@db.schema."stage.sub/dir"/v1', + "schema", + '@db.schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@schema."stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + '@schema."stage.sub/dir"/v1', + "schema", + '@schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + 'schema."stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + 'schema."stage.sub/dir"/v1', + "schema", + 'schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@"stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + '@"stage.sub/dir"/v1', + None, + '@"stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '"stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + '"stage.sub/dir"/v1', + None, + '"stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + 'db.schema."stage.sub/dir"/v1/more/and/more', + '"stage.sub/dir"/v1/more/and/more', + 'db.schema."stage.sub/dir"/v1/more/and/more', + "schema", + 'db.schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@db.schema."stage.sub/dir"/v1/more/and/more/', + '"stage.sub/dir"/v1/more/and/more', + '@db.schema."stage.sub/dir"/v1/more/and/more', + "schema", + '@db.schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@schema."stage.sub/dir"/v1/more/and/more/', + '"stage.sub/dir"/v1/more/and/more', + '@schema."stage.sub/dir"/v1/more/and/more', + "schema", + '@schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + 'schema."stage.sub/dir"/v1/more/and/more', + '"stage.sub/dir"/v1/more/and/more', + 'schema."stage.sub/dir"/v1/more/and/more', + "schema", + 'schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@"stage.sub/dir"/v1/more/and/more/', + '"stage.sub/dir"/v1/more/and/more', + '@"stage.sub/dir"/v1/more/and/more', + None, + '@"stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '"stage.sub/dir"/v1/more/and/more/', + '"stage.sub/dir"/v1/more/and/more', + '"stage.sub/dir"/v1/more/and/more', + None, + '"stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + 'db.schema."stage.sub/dir"', + '"stage.sub/dir"', + 'db.schema."stage.sub/dir"', + "schema", + 'db.schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@db.schema."stage.sub/dir"', + '"stage.sub/dir"', + '@db.schema."stage.sub/dir"', + "schema", + '@db.schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@schema."stage.sub/dir"', + '"stage.sub/dir"', + '@schema."stage.sub/dir"', + "schema", + '@schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + 'schema."stage.sub/dir"', + '"stage.sub/dir"', + 'schema."stage.sub/dir"', + "schema", + 'schema."stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@"stage.sub/dir"', + '"stage.sub/dir"', + '@"stage.sub/dir"', + None, + '@"stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '@"stage.sub/dir"', + '"stage.sub/dir"', + '@"stage.sub/dir"', + None, + '@"stage.sub/dir"', + '"stage.sub/dir"', + ), + ( + '"stage.sub/dir"', + '"stage.sub/dir"', + '"stage.sub/dir"', + None, + '"stage.sub/dir"', + '"stage.sub/dir"', + ), + ("@stage/v1/", "stage/v1", "@stage/v1", None, "@stage", "stage"), + ( + "@stage/v1/trailing/", + "stage/v1/trailing", + "@stage/v1/trailing", + None, + "@stage", + "stage", + ), + ( + "@stage/v1/file.name", + "stage/v1/file.name", + "@stage/v1/file.name", + None, + "@stage", + "stage", + ), + ( + "@stage/file.name", + "stage/file.name", + "@stage/file.name", + None, + "@stage", + "stage", + ), + ( + "@stage/v1/v2/file.name", + "stage/v1/v2/file.name", + "@stage/v1/v2/file.name", + None, + "@stage", + "stage", + ), + ( + "@stage/v1/v2/fi?e.name", + "stage/v1/v2/fi?e.name", + "@stage/v1/v2/fi?e.name", + None, + "@stage", + "stage", + ), + ( + "@stage/v1/v2/fi*e.name", + "stage/v1/v2/fi*e.name", + "@stage/v1/v2/fi*e.name", + None, + "@stage", + "stage", + ), ], ) def test_default_stage_path_parts( From c0b08266fd95c9a5be788ea9ddc2cb581b183496 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Fri, 6 Dec 2024 17:42:56 -0500 Subject: [PATCH 26/33] address comments --- RELEASE-NOTES.md | 2 +- .../nativeapp/entities/application.py | 4 +- .../nativeapp/entities/application_package.py | 9 +-- .../cli/_plugins/nativeapp/sf_sql_facade.py | 27 +++++---- .../test_application_package_entity.py | 1 - tests/nativeapp/test_event_sharing.py | 4 +- tests/nativeapp/test_run_processor.py | 56 +++++++++---------- tests/nativeapp/test_sf_sql_facade.py | 22 ++++---- tests/nativeapp/test_version_create.py | 8 +-- tests/stage/test_diff.py | 2 - 10 files changed, 68 insertions(+), 67 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index d316575ac8..f056a7866f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -19,7 +19,7 @@ ## Deprecations ## New additions -* Added an optional `stage_subdirectory` field to `application package` entity. This field specifies a subdirectory on the stage where the artifacts are to be found. The `manifest.yml` is at the root of this subdirectory. +* Added an optional `stage_subdirectory` field to `application package` entity. When specified, application artifacts are uploaded to this subdirectory instead of the root of the application package's stage. ## Fixes and improvements * Fixed crashes with older x86_64 Intel CPUs. diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application.py b/src/snowflake/cli/_plugins/nativeapp/entities/application.py index c2f80e4d32..5771cf03a9 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application.py @@ -609,7 +609,7 @@ def _upgrade_app( return get_snowflake_facade().upgrade_application( name=self.name, install_method=install_method, - stage_path_to_artifacts=stage_path.full_path, + path_to_version_directory=stage_path.full_path, debug_mode=self.debug, should_authorize_event_sharing=event_sharing.should_authorize_event_sharing(), role=self.role, @@ -659,7 +659,7 @@ def _create_app( name=self.name, package_name=package.name, install_method=install_method, - stage_path_to_artifacts=stage_path.full_path, + path_to_version_directory=stage_path.full_path, debug_mode=self.debug, should_authorize_event_sharing=event_sharing.should_authorize_event_sharing(), role=self.role, diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index 7522dad651..41cd117e20 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -2,6 +2,7 @@ import json import re +from functools import cached_property from pathlib import Path from textwrap import dedent from typing import List, Literal, Optional, Union @@ -126,7 +127,7 @@ class ApplicationPackageEntityModel(EntityModelBase): title="Path to manifest.yml. Unused and deprecated starting with Snowflake CLI 3.2", default="", ) - # PJ-TODO: does it need sanitation? + stage_subdirectory: Optional[str] = Field( title="Subfolder in stage", default="", @@ -218,7 +219,7 @@ def warehouse(self) -> str: def scratch_stage_path(self) -> DefaultStagePathParts: return DefaultStagePathParts(f"{self.name}.{self._entity_model.scratch_stage}") - @property + @cached_property def stage_path(self) -> DefaultStagePathParts: stage_fqn = f"{self.name}.{self._entity_model.stage}" subdir = self._entity_model.stage_subdirectory @@ -715,7 +716,7 @@ def add_new_version(self, version: str, label: str | None = None) -> None: get_snowflake_facade().create_version_in_package( role=self.role, package_name=self.name, - stage_path_to_artifacts=self.stage_path.full_path, + path_to_version_directory=self.stage_path.full_path, version=version, label=label, ) @@ -739,7 +740,7 @@ def add_new_patch_to_version( new_patch = get_snowflake_facade().add_patch_to_package_version( role=self.role, package_name=self.name, - stage_path_to_artifacts=self.stage_path.full_path, + path_to_version_directory=self.stage_path.full_path, version=version, patch=patch, label=label, diff --git a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py index 967067c41a..65ca1d7055 100644 --- a/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +++ b/src/snowflake/cli/_plugins/nativeapp/sf_sql_facade.py @@ -39,6 +39,7 @@ UserScriptError, handle_unclassified_error, ) +from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.constants import ObjectType from snowflake.cli.api.errno import ( @@ -243,7 +244,7 @@ def get_account_event_table(self, role: str | None = None) -> str | None: def create_version_in_package( self, package_name: str, - stage_path_to_artifacts: str, + path_to_version_directory: str, version: str, label: str | None = None, role: str | None = None, @@ -251,7 +252,7 @@ def create_version_in_package( """ Creates a new version in an existing application package. @param package_name: Name of the application package to alter. - @param stage_path_to_artifacts: Path to artifacts on the stage to create a version from. + @param path_to_version_directory: Path to artifacts on the stage to create a version from. @param version: Version name to create. @param [Optional] role: Switch to this role while executing create version. @param [Optional] label: Label for this version, visible to consumers. @@ -264,11 +265,12 @@ def create_version_in_package( with_label_cause = ( f"\nlabel={to_string_literal(label)}" if label is not None else "" ) + using_clause = StageManager.quote_stage_name(path_to_version_directory) add_version_query = dedent( f"""\ alter application package {package_name} add version {version} - using @{stage_path_to_artifacts}{with_label_cause} + using {using_clause}{with_label_cause} """ ) with self._use_role_optional(role): @@ -283,7 +285,7 @@ def create_version_in_package( def add_patch_to_package_version( self, package_name: str, - stage_path_to_artifacts: str, + path_to_version_directory: str, version: str, patch: int | None = None, label: str | None = None, @@ -292,7 +294,7 @@ def add_patch_to_package_version( """ Add a new patch, optionally a custom one, to an existing version in an application package. @param package_name: Name of the application package to alter. - @param stage_path_to_artifacts: Path to artifacts on the stage to create a version from. + @param path_to_version_directory: Path to artifacts on the stage to create a version from. @param version: Version name to create. @param [Optional] patch: Patch number to create. @param [Optional] label: Label for this patch, visible to consumers. @@ -309,11 +311,12 @@ def add_patch_to_package_version( f"\nlabel={to_string_literal(label)}" if label is not None else "" ) patch_query = f"{patch}" if patch else "" + using_clause = StageManager.quote_stage_name(path_to_version_directory) add_patch_query = dedent( f"""\ alter application package {package_name} add patch {patch_query} for version {version} - using @{stage_path_to_artifacts}{with_label_clause} + using {using_clause}{with_label_clause} """ ) with self._use_role_optional(role): @@ -581,7 +584,7 @@ def upgrade_application( self, name: str, install_method: SameAccountInstallMethod, - stage_path_to_artifacts: str, + path_to_version_directory: str, role: str, warehouse: str, debug_mode: bool | None, @@ -592,7 +595,7 @@ def upgrade_application( @param name: Name of the application object @param install_method: Method of installing the application - @param stage_path_to_artifacts: Path to directory in stage housing the application artifacts + @param path_to_version_directory: Path to directory in stage housing the application artifacts @param role: Role to use when creating the application and provider-side objects @param warehouse: Warehouse which is required to create an application object @param debug_mode: Whether to enable debug mode; None means not explicitly enabled or disabled @@ -607,7 +610,7 @@ def upgrade_application( with self._use_role_optional(role), self._use_warehouse_optional(warehouse): try: - using_clause = install_method.using_clause(stage_path_to_artifacts) + using_clause = install_method.using_clause(path_to_version_directory) upgrade_cursor = self._sql_executor.execute_query( f"alter application {name} upgrade {using_clause}", ) @@ -677,7 +680,7 @@ def create_application( name: str, package_name: str, install_method: SameAccountInstallMethod, - stage_path_to_artifacts: str, + path_to_version_directory: str, role: str, warehouse: str, debug_mode: bool | None, @@ -690,7 +693,7 @@ def create_application( @param name: Name of the application object @param package_name: Name of the application package to install the application from @param install_method: Method of installing the application - @param stage_path_to_artifacts: Path to directory in stage housing the application artifacts + @param path_to_version_directory: Path to directory in stage housing the application artifacts @param role: Role to use when creating the application and provider-side objects @param warehouse: Warehouse which is required to create an application object @param debug_mode: Whether to enable debug mode; None means not explicitly enabled or disabled @@ -712,7 +715,7 @@ def create_application( ) authorize_telemetry_clause = f" AUTHORIZE_TELEMETRY_EVENT_SHARING = {str(should_authorize_event_sharing).upper()}" - using_clause = install_method.using_clause(stage_path_to_artifacts) + using_clause = install_method.using_clause(path_to_version_directory) with self._use_role_optional(role), self._use_warehouse_optional(warehouse): try: create_cursor = self._sql_executor.execute_query( diff --git a/tests/nativeapp/test_application_package_entity.py b/tests/nativeapp/test_application_package_entity.py index f3c29eed1d..2f60682eb8 100644 --- a/tests/nativeapp/test_application_package_entity.py +++ b/tests/nativeapp/test_application_package_entity.py @@ -49,7 +49,6 @@ def _get_app_pkg_entity(project_directory, package_overrides=None): with project_directory("workspaces_simple") as project_root: with Path(project_root / "snowflake.yml").open() as definition_file_path: project_definition = yaml.safe_load(definition_file_path) - # project_definition["entities"]["pkg"]["stage_subdirectory"] = None project_definition["entities"]["pkg"] = dict( project_definition["entities"]["pkg"], **(package_overrides or {}) ) diff --git a/tests/nativeapp/test_event_sharing.py b/tests/nativeapp/test_event_sharing.py index 40effe96f6..c9b0af8668 100644 --- a/tests/nativeapp/test_event_sharing.py +++ b/tests/nativeapp/test_event_sharing.py @@ -302,7 +302,7 @@ def _setup_mocks_for_create_app( install_method=SameAccountInstallMethod.release_directive() if is_prod else SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=stage_path_to_artifacts, + path_to_version_directory=stage_path_to_artifacts, debug_mode=None, should_authorize_event_sharing=expected_authorize_telemetry_flag, role="app_role", @@ -406,7 +406,7 @@ def _setup_mocks_for_upgrade_app( install_method=SameAccountInstallMethod.release_directive() if is_prod else SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=stage_path_to_artifacts, + path_to_version_directory=stage_path_to_artifacts, debug_mode=None, should_authorize_event_sharing=expected_authorize_telemetry_flag, role="app_role", diff --git a/tests/nativeapp/test_run_processor.py b/tests/nativeapp/test_run_processor.py index ace2b9039d..0e659025aa 100644 --- a/tests/nativeapp/test_run_processor.py +++ b/tests/nativeapp/test_run_processor.py @@ -270,7 +270,7 @@ def test_create_dev_app_w_warehouse_access_exception( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -341,7 +341,7 @@ def test_create_dev_app_create_new_w_no_additional_privileges( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -414,7 +414,7 @@ def test_create_or_upgrade_dev_app_with_warning( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -428,7 +428,7 @@ def test_create_or_upgrade_dev_app_with_warning( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -482,7 +482,7 @@ def test_create_dev_app_create_new_with_additional_privileges( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -559,7 +559,7 @@ def test_create_dev_app_create_new_w_missing_warehouse_exception( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -670,7 +670,7 @@ def test_create_dev_app_incorrect_owner( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -723,7 +723,7 @@ def test_create_dev_app_no_diff_changes( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -779,7 +779,7 @@ def test_create_dev_app_w_diff_changes( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -903,7 +903,7 @@ def test_create_dev_app_create_new_quoted( name='"My Application"', package_name='"My Package"', install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts='"My Package".app_src.stage', + path_to_version_directory='"My Package".app_src.stage', debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -960,7 +960,7 @@ def test_create_dev_app_create_new_quoted_override( name='"My Application"', package_name='"My Package"', install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts='"My Package".app_src.stage', + path_to_version_directory='"My Package".app_src.stage', debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1042,7 +1042,7 @@ def test_create_dev_app_recreate_app_when_orphaned( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1054,7 +1054,7 @@ def test_create_dev_app_recreate_app_when_orphaned( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1181,7 +1181,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1194,7 +1194,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1318,7 +1318,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade_unknown_obje mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1330,7 +1330,7 @@ def test_create_dev_app_recreate_app_when_orphaned_requires_cascade_unknown_obje name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1475,7 +1475,7 @@ def test_upgrade_app_incorrect_owner( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1530,7 +1530,7 @@ def test_upgrade_app_succeeds( mock_sql_facade_upgrade_application.assert_called_once_with( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1589,7 +1589,7 @@ def test_upgrade_app_fails_generic_error( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1670,7 +1670,7 @@ def test_upgrade_app_fails_upgrade_restriction_error( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1750,7 +1750,7 @@ def test_versioned_app_upgrade_to_unversioned( mock_sql_facade_upgrade_application.assert_called_once_with( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1761,7 +1761,7 @@ def test_versioned_app_upgrade_to_unversioned( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1869,7 +1869,7 @@ def test_upgrade_app_fails_drop_fails( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1950,7 +1950,7 @@ def test_upgrade_app_recreate_app( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -1962,7 +1962,7 @@ def test_upgrade_app_recreate_app( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2131,7 +2131,7 @@ def test_upgrade_app_recreate_app_from_version( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.versioned_dev("v1"), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2143,7 +2143,7 @@ def test_upgrade_app_recreate_app_from_version( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.versioned_dev("v1"), - stage_path_to_artifacts=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, diff --git a/tests/nativeapp/test_sf_sql_facade.py b/tests/nativeapp/test_sf_sql_facade.py index e275507372..64eefad3d7 100644 --- a/tests/nativeapp/test_sf_sql_facade.py +++ b/tests/nativeapp/test_sf_sql_facade.py @@ -1787,7 +1787,7 @@ def test_upgrade_application_unversioned( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, @@ -1838,7 +1838,7 @@ def test_upgrade_application_version_and_patch( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.versioned_dev("3", 2), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -1885,7 +1885,7 @@ def test_upgrade_application_from_release_directive( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -1931,7 +1931,7 @@ def test_upgrade_application_converts_expected_programmingerrors_to_user_errors( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -1996,7 +1996,7 @@ def test_upgrade_application_special_message_for_event_sharing_error( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.versioned_dev("v1"), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=False, should_authorize_event_sharing=False, role=role, @@ -2045,7 +2045,7 @@ def test_upgrade_application_converts_unexpected_programmingerrors_to_unclassifi sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -2096,7 +2096,7 @@ def test_create_application_with_minimal_clauses( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, @@ -2145,7 +2145,7 @@ def test_create_application_with_all_clauses( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.unversioned_dev(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -2198,7 +2198,7 @@ def test_create_application_converts_expected_programmingerrors_to_user_errors( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, @@ -2257,7 +2257,7 @@ def test_create_application_special_message_for_event_sharing_error( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.versioned_dev("3", 1), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=False, should_authorize_event_sharing=False, role=role, @@ -2315,7 +2315,7 @@ def test_create_application_converts_unexpected_programmingerrors_to_unclassifie name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_path_to_artifacts=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, diff --git a/tests/nativeapp/test_version_create.py b/tests/nativeapp/test_version_create.py index 0e4c689bd2..db3e1a02d4 100644 --- a/tests/nativeapp/test_version_create.py +++ b/tests/nativeapp/test_version_create.py @@ -699,7 +699,7 @@ def test_manifest_version_info_not_used( mock_create_version.assert_called_with( role=role, package_name="app_pkg", - stage_path_to_artifacts=f"app_pkg.{stage}", + path_to_version_directory=f"app_pkg.{stage}", version=version_cli, label="", ) @@ -765,7 +765,7 @@ def test_manifest_patch_is_not_used( mock_create_patch.assert_called_with( role=role, package_name="app_pkg", - stage_path_to_artifacts=f"app_pkg.{stage}", + path_to_version_directory=f"app_pkg.{stage}", version=version_cli, patch=patch, # ensure empty label is used to replace label from manifest.yml @@ -840,7 +840,7 @@ def test_version_from_manifest( mock_create_patch.assert_called_with( role=role, package_name="app_pkg", - stage_path_to_artifacts=f"app_pkg.{stage}", + path_to_version_directory=f"app_pkg.{stage}", version="manifest_version", patch=manifest_patch, label=cli_label if cli_label is not None else manifest_label, @@ -914,7 +914,7 @@ def test_patch_from_manifest( mock_create_patch.assert_called_with( role=role, package_name="app_pkg", - stage_path_to_artifacts=f"app_pkg.{stage}", + path_to_version_directory=f"app_pkg.{stage}", version="manifest_version", # cli patch overrides the manifest patch=cli_patch, diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index 6eee0b1474..3c28629129 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -89,7 +89,6 @@ def stage_contents( ] -# PJ - TODO add tests for following cases with subdir @mock.patch(f"{STAGE_MANAGER}.list_files") def test_empty_stage(mock_list, mock_cursor): mock_list.return_value = mock_cursor(rows=[], columns=STAGE_LS_COLUMNS) @@ -276,7 +275,6 @@ def test_put_files_on_stage(mock_put, overwrite_param): assert mock_put.mock_calls == expected -# PJ - TODO add test here def test_build_md5_map(mock_cursor): actual = build_md5_map( mock_cursor( From 4ec1071d7b42f0716a4e208ef8edaa74ff16bc61 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Fri, 6 Dec 2024 21:03:26 -0500 Subject: [PATCH 27/33] make for quoted identifiers in diff --- src/snowflake/cli/_plugins/stage/diff.py | 14 ++++++++------ tests/stage/test_diff.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index 83870be20b..f8bc707dee 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -83,24 +83,26 @@ def enumerate_files(path: Path) -> List[Path]: return paths -def relative_to_stage_path(path: str, stage_path: str) -> StagePathType: +def relative_to_stage_path(path: str, stage_subirectory: str) -> StagePathType: """ @param path: file path on the stage. - @param stage_path: root of stage. stage_name/[optionally/other/directories] + @param stage_subirectory: subdirectory of stage. @return: path of file relative to the stage_path """ - return StagePathType(path).relative_to(stage_path) + wo_stage_name = StagePathType(*path.split("/")[1:]) + relative_path = str(wo_stage_name).removeprefix(stage_subirectory).lstrip("/") + return StagePathType(relative_path) def build_md5_map( - list_stage_cursor: DictCursor, stage_path: str + list_stage_cursor: DictCursor, stage_subirectory: str ) -> Dict[StagePathType, Optional[str]]: """ Returns a mapping of file paths to their md5sums. File paths are relative to the stage_path. """ all_files = list_stage_cursor.fetchall() return { - relative_to_stage_path(file["name"], stage_path): file["md5"] + relative_to_stage_path(file["name"], stage_subirectory): file["md5"] for file in all_files } @@ -131,7 +133,7 @@ def compute_stage_diff(local_root: Path, stage_path: StagePathParts) -> DiffResu remote_files = stage_manager.list_files(stage_path.full_path) # Create a mapping from remote_file path to file's md5sum. Path is relative to stage_name/directory. - remote_md5 = build_md5_map(remote_files, stage_path.path) + remote_md5 = build_md5_map(remote_files, stage_path.directory) result: DiffResult = DiffResult() diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index 3c28629129..b5695ed78c 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -281,7 +281,7 @@ def test_build_md5_map(mock_cursor): rows=stage_contents(FILE_CONTENTS), columns=STAGE_LS_COLUMNS, ), - "stage", + "", ) expected = { From 7f30feadbb7a39e66fe71014f84843e28f1e2a22 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Thu, 12 Dec 2024 11:11:29 -0500 Subject: [PATCH 28/33] make diff work with all identifiers, update docstrings --- .../cli/_plugins/nativeapp/commands.py | 3 ++- .../nativeapp/entities/application_package.py | 9 +++---- src/snowflake/cli/_plugins/stage/diff.py | 24 +++++++++++-------- src/snowflake/cli/_plugins/stage/manager.py | 7 ++++++ .../test_application_package_entity.py | 4 ++-- tests/nativeapp/test_manager.py | 14 +++++------ tests/nativeapp/test_run_processor.py | 2 +- 7 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index dead1a846f..68058a26ef 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -58,6 +58,7 @@ ObjectResult, StreamResult, ) +from snowflake.cli.api.project.util import same_identifiers from typing_extensions import Annotated app = SnowTyperFactory( @@ -249,7 +250,7 @@ def app_teardown( for package_entity in project.get_entities_by_type( ApplicationPackageEntityModel.get_type() ).values() - if package_entity.identifier == app_package_entity.identifier + if same_identifiers(package_entity.identifier, app_package_entity.identifier) ] for app_entity in project.get_entities_by_type( diff --git a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py index aedf854e88..7d6d63c708 100644 --- a/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py +++ b/src/snowflake/cli/_plugins/nativeapp/entities/application_package.py @@ -135,7 +135,7 @@ class ApplicationPackageEntityModel(EntityModelBase): ) stage_subdirectory: Optional[str] = Field( - title="Subfolder in stage", + title="Subfolder in stage to upload the artifacts to, instead of the root of the application package's stage", default="", ) @@ -223,14 +223,15 @@ def warehouse(self) -> str: @property def scratch_stage_path(self) -> DefaultStagePathParts: - return DefaultStagePathParts(f"{self.name}.{self._entity_model.scratch_stage}") + return DefaultStagePathParts.from_fqn( + f"{self.name}.{self._entity_model.scratch_stage}" + ) @cached_property def stage_path(self) -> DefaultStagePathParts: stage_fqn = f"{self.name}.{self._entity_model.stage}" subdir = self._entity_model.stage_subdirectory - full_path = f"{stage_fqn}/{subdir}" if subdir else stage_fqn - return DefaultStagePathParts(full_path) + return DefaultStagePathParts.from_fqn(stage_fqn, subdir) @property def post_deploy_hooks(self) -> list[PostDeployHook] | None: diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index f8bc707dee..763b1c161b 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -23,6 +23,7 @@ from snowflake.cli.api.exceptions import ( SnowflakeSQLExecutionError, ) +from snowflake.cli.api.project.util import unquote_identifier from snowflake.connector.cursor import DictCursor from .manager import StageManager, StagePathParts @@ -83,26 +84,29 @@ def enumerate_files(path: Path) -> List[Path]: return paths -def relative_to_stage_path(path: str, stage_subirectory: str) -> StagePathType: +def relative_to_stage_path(path: str, stage_path: StagePathParts) -> StagePathType: """ @param path: file path on the stage. - @param stage_subirectory: subdirectory of stage. - @return: path of file relative to the stage_path + @param stage_path: stage path object. + @return: path of the file relative to the stage and subdirectory """ - wo_stage_name = StagePathType(*path.split("/")[1:]) - relative_path = str(wo_stage_name).removeprefix(stage_subirectory).lstrip("/") + # path is returned from a SQL call so it's unquoted. Unquote stage_path identifiers to match. + stage_name = unquote_identifier(stage_path.stage_name) + stage_subdirectory = unquote_identifier(stage_path.directory) + path_wo_stage_name = path.removeprefix(stage_name) + relative_path = path_wo_stage_name.removeprefix(stage_subdirectory).lstrip("/") return StagePathType(relative_path) def build_md5_map( - list_stage_cursor: DictCursor, stage_subirectory: str + list_stage_cursor: DictCursor, stage_path: StagePathParts ) -> Dict[StagePathType, Optional[str]]: """ - Returns a mapping of file paths to their md5sums. File paths are relative to the stage_path. + Returns a mapping of file paths to their md5sums. File paths are relative to the stage and subdirectory. """ all_files = list_stage_cursor.fetchall() return { - relative_to_stage_path(file["name"], stage_subirectory): file["md5"] + relative_to_stage_path(file["name"], stage_path): file["md5"] for file in all_files } @@ -126,14 +130,14 @@ def preserve_from_diff( def compute_stage_diff(local_root: Path, stage_path: StagePathParts) -> DiffResult: """ - Diffs the files in the local_root with files in the stage path that is stage_path_parts's full_path. + Diffs the files in the local_root with files in the stage path that is stage_path's full_path. """ stage_manager = StageManager() local_files = enumerate_files(local_root) remote_files = stage_manager.list_files(stage_path.full_path) # Create a mapping from remote_file path to file's md5sum. Path is relative to stage_name/directory. - remote_md5 = build_md5_map(remote_files, stage_path.directory) + remote_md5 = build_md5_map(remote_files, stage_path) result: DiffResult = DiffResult() diff --git a/src/snowflake/cli/_plugins/stage/manager.py b/src/snowflake/cli/_plugins/stage/manager.py index e7b0625595..9ae379ff6e 100644 --- a/src/snowflake/cli/_plugins/stage/manager.py +++ b/src/snowflake/cli/_plugins/stage/manager.py @@ -147,6 +147,13 @@ def __init__(self, stage_path: str): self.stage_name = stage_name self.is_directory = True if stage_path.endswith("/") else False + @classmethod + def from_fqn( + cls, stage_fqn: str, subdir: str | None = None + ) -> DefaultStagePathParts: + full_path = f"{stage_fqn}/{subdir}" if subdir else stage_fqn + return cls(full_path) + @property def path(self) -> str: return f"{self.stage_name.rstrip('/')}/{self.directory}".rstrip("/") diff --git a/tests/nativeapp/test_application_package_entity.py b/tests/nativeapp/test_application_package_entity.py index 7b23ce5c31..d0e1d1f24d 100644 --- a/tests/nativeapp/test_application_package_entity.py +++ b/tests/nativeapp/test_application_package_entity.py @@ -210,7 +210,7 @@ def test_deploy( role="app_role", prune=False, recursive=False, - stage_path=DefaultStagePathParts("pkg.app_src.stage"), + stage_path=DefaultStagePathParts.from_fqn("pkg.app_src.stage"), local_paths_to_sync=["a/b", "c"], print_diff=True, ) @@ -311,7 +311,7 @@ def test_deploy_w_stage_subdir( role="app_role", prune=False, recursive=False, - stage_path=DefaultStagePathParts("pkg.app_src.stage/v1"), + stage_path=DefaultStagePathParts.from_fqn("pkg.app_src.stage", "v1"), local_paths_to_sync=["a/b", "c"], print_diff=True, ) diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index c66f6eb51b..f0a143c304 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -151,7 +151,7 @@ def test_sync_deploy_root_with_stage( role="new_role", prune=True, recursive=True, - stage_path=DefaultStagePathParts(stage_fqn), + stage_path=DefaultStagePathParts.from_fqn(stage_fqn), ) mock_stage_exists.assert_called_once_with(stage_fqn) @@ -160,7 +160,7 @@ def test_sync_deploy_root_with_stage( mock_create_stage.assert_called_once_with(stage_fqn) mock_compute_stage_diff.assert_called_once_with( local_root=dm.project_root / pkg_model.deploy_root, - stage_path=DefaultStagePathParts("app_pkg.app_src.stage"), + stage_path=DefaultStagePathParts.from_fqn("app_pkg.app_src.stage"), ) mock_local_diff_with_stage.assert_called_once_with( role="new_role", @@ -215,7 +215,7 @@ def test_sync_deploy_root_with_stage_subdir( role="new_role", prune=True, recursive=True, - stage_path=DefaultStagePathParts(stage_full_path), + stage_path=DefaultStagePathParts.from_fqn(stage_fqn, "v1"), ) mock_stage_exists.assert_called_once_with(stage_fqn) @@ -224,7 +224,7 @@ def test_sync_deploy_root_with_stage_subdir( mock_create_stage.assert_called_once_with(stage_fqn) mock_compute_stage_diff.assert_called_once_with( local_root=dm.project_root / pkg_model.deploy_root, - stage_path=DefaultStagePathParts(stage_full_path), + stage_path=DefaultStagePathParts.from_fqn(stage_fqn, "v1"), ) mock_local_diff_with_stage.assert_called_once_with( role="new_role", @@ -285,7 +285,7 @@ def test_sync_deploy_root_with_stage_prune( role="new_role", prune=prune, recursive=True, - stage_path=DefaultStagePathParts(stage_fqn), + stage_path=DefaultStagePathParts.from_fqn(stage_fqn), ) if expected_warn: @@ -1446,7 +1446,7 @@ def test_validate_use_scratch_stage(mock_execute, mock_deploy, temp_dir, mock_cu paths=[], print_diff=False, validate=False, - stage_path=DefaultStagePathParts( + stage_path=DefaultStagePathParts.from_fqn( f"{pkg_model.fqn.name}.{pkg_model.scratch_stage}" ), interactive=False, @@ -1524,7 +1524,7 @@ def test_validate_failing_drops_scratch_stage( paths=[], print_diff=False, validate=False, - stage_path=DefaultStagePathParts( + stage_path=DefaultStagePathParts.from_fqn( f"{pkg_model.fqn.name}.{pkg_model.scratch_stage}" ), interactive=False, diff --git a/tests/nativeapp/test_run_processor.py b/tests/nativeapp/test_run_processor.py index 0e659025aa..c3edd992ac 100644 --- a/tests/nativeapp/test_run_processor.py +++ b/tests/nativeapp/test_run_processor.py @@ -166,7 +166,7 @@ def _create_or_upgrade_app( return app.create_or_upgrade_app( package=pkg, - stage_path=DefaultStagePathParts(stage_fqn), + stage_path=DefaultStagePathParts.from_fqn(stage_fqn), install_method=install_method, policy=policy, interactive=interactive, From 1921805ba03be7d8a2f83bb5a3ee6f2a42a459fa Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Thu, 12 Dec 2024 12:18:13 -0500 Subject: [PATCH 29/33] update tests --- src/snowflake/cli/_plugins/stage/diff.py | 7 ++++--- tests/nativeapp/test_sf_sql_facade.py | 8 ++++---- tests/nativeapp/test_version_create.py | 2 +- tests/stage/test_diff.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/snowflake/cli/_plugins/stage/diff.py b/src/snowflake/cli/_plugins/stage/diff.py index 763b1c161b..7d74f5649c 100644 --- a/src/snowflake/cli/_plugins/stage/diff.py +++ b/src/snowflake/cli/_plugins/stage/diff.py @@ -91,9 +91,10 @@ def relative_to_stage_path(path: str, stage_path: StagePathParts) -> StagePathTy @return: path of the file relative to the stage and subdirectory """ # path is returned from a SQL call so it's unquoted. Unquote stage_path identifiers to match. - stage_name = unquote_identifier(stage_path.stage_name) - stage_subdirectory = unquote_identifier(stage_path.directory) - path_wo_stage_name = path.removeprefix(stage_name) + # Stage is always returned in lower-case from ls SQL request + stage_name = unquote_identifier(stage_path.stage_name).lower() + stage_subdirectory = stage_path.directory + path_wo_stage_name = path.removeprefix(stage_name).lstrip("/") relative_path = path_wo_stage_name.removeprefix(stage_subdirectory).lstrip("/") return StagePathType(relative_path) diff --git a/tests/nativeapp/test_sf_sql_facade.py b/tests/nativeapp/test_sf_sql_facade.py index ae570a7b20..649c4b1157 100644 --- a/tests/nativeapp/test_sf_sql_facade.py +++ b/tests/nativeapp/test_sf_sql_facade.py @@ -3318,7 +3318,7 @@ def test_create_version_in_package( package_name=package_name, version=version, role=role, - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, ) @@ -3358,7 +3358,7 @@ def test_create_version_in_package_with_label( package_name=package_name, version=version, role=role, - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, label=label, ) @@ -3397,7 +3397,7 @@ def test_create_version_with_special_characters( package_name=package_name, version=version, role=role, - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, ) @@ -3418,7 +3418,7 @@ def test_create_version_in_package_with_error( package_name=package_name, version=version, role=role, - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, ) diff --git a/tests/nativeapp/test_version_create.py b/tests/nativeapp/test_version_create.py index 1e85b7ab9e..69a88af995 100644 --- a/tests/nativeapp/test_version_create.py +++ b/tests/nativeapp/test_version_create.py @@ -163,7 +163,7 @@ def test_add_version( mock_create_version.assert_called_once_with( package_name="app_pkg", version=version, - stage_fqn=f"app_pkg.{pkg_model.stage}", + path_to_version_directory=f"app_pkg.{pkg_model.stage}", role="package_role", label=None, ) diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index b5695ed78c..8633bcc747 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -281,7 +281,7 @@ def test_build_md5_map(mock_cursor): rows=stage_contents(FILE_CONTENTS), columns=STAGE_LS_COLUMNS, ), - "", + DefaultStagePathParts.from_fqn("stage"), ) expected = { From d84a59c1d99838f56e246ab8dd0e5d34c9adb75f Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Fri, 13 Dec 2024 11:33:46 -0500 Subject: [PATCH 30/33] fix tests --- tests/nativeapp/test_application_package_entity.py | 4 ++-- tests/nativeapp/test_run_processor.py | 8 ++++---- tests/nativeapp/test_sf_sql_facade.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/nativeapp/test_application_package_entity.py b/tests/nativeapp/test_application_package_entity.py index 15029c2a9a..c07eb3c607 100644 --- a/tests/nativeapp/test_application_package_entity.py +++ b/tests/nativeapp/test_application_package_entity.py @@ -102,7 +102,7 @@ def package_with_subdir_factory(): def test_bundle_with_subdir(project_directory): package_with_subdir_factory() app_pkg, bundle_ctx, mock_console = _get_app_pkg_entity( - project_directory, {"stage_subdirectory": "v1"} + project_directory, package_overrides={"stage_subdirectory": "v1"} ) bundle_result = app_pkg.action_bundle(bundle_ctx) @@ -291,7 +291,7 @@ def test_deploy_w_stage_subdir( mock_execute.side_effect = side_effects app_pkg, bundle_ctx, mock_console = _get_app_pkg_entity( - project_directory, {"stage_subdirectory": "v1"} + project_directory, package_overrides={"stage_subdirectory": "v1"} ) app_pkg.action_deploy( diff --git a/tests/nativeapp/test_run_processor.py b/tests/nativeapp/test_run_processor.py index 9a6ff60799..b3c7fb7530 100644 --- a/tests/nativeapp/test_run_processor.py +++ b/tests/nativeapp/test_run_processor.py @@ -2298,7 +2298,7 @@ def test_run_app_from_release_directive_with_channel( mock.call( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2311,7 +2311,7 @@ def test_run_app_from_release_directive_with_channel( name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2668,7 +2668,7 @@ def test_run_app_from_release_directive_with_default_channel_when_release_channe mock_sql_facade_upgrade_application.assert_called_once_with( name=DEFAULT_APP_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, @@ -2679,7 +2679,7 @@ def test_run_app_from_release_directive_with_default_channel_when_release_channe name=DEFAULT_APP_ID, package_name=DEFAULT_PKG_ID, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=DEFAULT_STAGE_FQN, + path_to_version_directory=DEFAULT_STAGE_FQN, debug_mode=True, should_authorize_event_sharing=None, role=DEFAULT_ROLE, diff --git a/tests/nativeapp/test_sf_sql_facade.py b/tests/nativeapp/test_sf_sql_facade.py index 4860e52bb4..cea7e87b8c 100644 --- a/tests/nativeapp/test_sf_sql_facade.py +++ b/tests/nativeapp/test_sf_sql_facade.py @@ -2104,7 +2104,7 @@ def test_upgrade_application_with_release_channel_same_as_app_properties( sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -2136,7 +2136,7 @@ def test_upgrade_application_with_release_channel_not_same_as_app_properties_the sql_facade.upgrade_application( name=app_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=True, should_authorize_event_sharing=True, role=role, @@ -2469,7 +2469,7 @@ def test_create_application_with_release_channel( name=app_name, package_name=pkg_name, install_method=SameAccountInstallMethod.release_directive(), - stage_fqn=stage_fqn, + path_to_version_directory=stage_fqn, debug_mode=None, should_authorize_event_sharing=None, role=role, From d865ac8668708aede68234421e75f2659ebaa436 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Mon, 16 Dec 2024 19:20:41 -0500 Subject: [PATCH 31/33] update commands --- src/snowflake/cli/_plugins/nativeapp/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index f16617c873..a2e530e2fd 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -257,7 +257,7 @@ def app_teardown( for package_entity in project.get_entities_by_type( ApplicationPackageEntityModel.get_type() ).values() - if same_identifiers(package_entity.identifier, app_package_entity.identifier) + if same_identifiers(package_entity.fqn.name, app_package_entity.fqn.name) ] for app_entity in project.get_entities_by_type( From aa80ac4154ebc0df412e1132fb29b25bb4289166 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 17 Dec 2024 11:24:38 -0500 Subject: [PATCH 32/33] replace test data with factory --- tests_integration/nativeapp/test_bundle.py | 15 +- tests_integration/nativeapp/test_deploy.py | 39 +++-- tests_integration/nativeapp/test_version.py | 60 ++++++-- .../napp_stage_subdirs/app/v1/README.md | 3 - .../napp_stage_subdirs/app/v1/manifest.yml | 11 -- .../app/v1/setup_script.sql | 5 - .../napp_stage_subdirs/app/v2/README.md | 3 - .../napp_stage_subdirs/app/v2/manifest.yml | 11 -- .../app/v2/setup_script.sql | 5 - .../projects/napp_stage_subdirs/snowflake.yml | 31 ---- .../app/v1/README.md | 3 - .../app/v1/manifest.yml | 11 -- .../app/v1/module-echo-v1/echo-v1.py | 13 -- .../app/v1/setup_script.sql | 5 - .../app/v2/README.md | 3 - .../app/v2/manifest.yml | 11 -- .../app/v2/module-echo-v2/echo-v2.py | 13 -- .../app/v2/setup_script.sql | 5 - .../snowflake.yml | 38 ----- .../testing_utils/project_fixtures.py | 137 ++++++++++++++++++ 20 files changed, 221 insertions(+), 201 deletions(-) delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql delete mode 100644 tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml create mode 100644 tests_integration/testing_utils/project_fixtures.py diff --git a/tests_integration/nativeapp/test_bundle.py b/tests_integration/nativeapp/test_bundle.py index 5c5eb933ba..2bcfa1bd6a 100644 --- a/tests_integration/nativeapp/test_bundle.py +++ b/tests_integration/nativeapp/test_bundle.py @@ -23,6 +23,7 @@ from tests_integration.testing_utils import ( assert_that_result_failed_with_message_containing, ) +from tests_integration.testing_utils.project_fixtures import * @pytest.fixture @@ -323,10 +324,13 @@ def test_nativeapp_bundle_deletes_existing_deploy_root(template_setup): @pytest.mark.integration -def test_nativeapp_can_bundle_with_subdirs(runner, nativeapp_project_directory): +def test_nativeapp_can_bundle_with_subdirs( + runner, nativeapp_teardown, setup_v2_project_w_subdir +): command = "app bundle --package-entity-id=pkg_v1" subdir = "v1" - with nativeapp_project_directory("napp_stage_subdirs") as project_root: + project_name, project_root = setup_v2_project_w_subdir() + with nativeapp_teardown(): result = runner.invoke_json(split(command)) assert result.exit_code == 0 @@ -337,9 +341,12 @@ def test_nativeapp_can_bundle_with_subdirs(runner, nativeapp_project_directory): @pytest.mark.integration -def test_nativeapp_bundle_subdirs_dont_overwrite(runner, nativeapp_project_directory): +def test_nativeapp_bundle_subdirs_dont_overwrite( + runner, nativeapp_teardown, setup_v2_project_w_subdir_w_snowpark +): + project_name, project_root = setup_v2_project_w_subdir_w_snowpark() - with nativeapp_project_directory("napp_stage_subdirs_w_snowpark") as project_root: + with nativeapp_teardown(): result_1 = runner.invoke_json(split("app bundle --package-entity-id=pkg_v1")) assert result_1.exit_code == 0 diff --git a/tests_integration/nativeapp/test_deploy.py b/tests_integration/nativeapp/test_deploy.py index be29b3e10e..d0879ee95b 100644 --- a/tests_integration/nativeapp/test_deploy.py +++ b/tests_integration/nativeapp/test_deploy.py @@ -17,12 +17,13 @@ from snowflake.cli.api.project.util import TEST_RESOURCE_SUFFIX_VAR from tests.nativeapp.utils import touch - +from tests_integration.testing_utils.project_fixtures import * from tests.project.fixtures import * from tests_integration.test_utils import ( contains_row_with, not_contains_row_with, row_from_snowflake_session, + row_from_cursor, ) from tests_integration.testing_utils import ( assert_that_result_failed_with_message_containing, @@ -108,7 +109,7 @@ def test_nativeapp_deploy( @pytest.mark.integration def test_nativeapp_deploy_w_stage_subdir( - nativeapp_project_directory, + nativeapp_teardown, runner, snowflake_session, default_username, @@ -116,9 +117,13 @@ def test_nativeapp_deploy_w_stage_subdir( sanitize_deploy_output, snapshot, print_paths_as_posix, + setup_v2_project_w_subdir, ): - project_name = "stage_w_subdirs_pkg" - with nativeapp_project_directory("napp_stage_subdirs"): + ( + project_name, + _, + ) = setup_v2_project_w_subdir() # make this a fixture, don't pass temp_dir + with nativeapp_teardown(): result = runner.invoke_with_connection( split("app deploy --package-entity-id=pkg_v1") ) @@ -127,8 +132,9 @@ def test_nativeapp_deploy_w_stage_subdir( assert sanitize_deploy_output(result.output) == snapshot # package exist - package_name = f"{project_name}_{default_username}{resource_suffix}".upper() - app_name = f"{project_name}_{default_username}{resource_suffix}".upper() + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() + app_name = f"{project_name}_app_v1_{default_username}{resource_suffix}".upper() + assert contains_row_with( row_from_snowflake_session( snowflake_session.execute_string( @@ -139,7 +145,7 @@ def test_nativeapp_deploy_w_stage_subdir( ) # manifest file exists - stage_name = "app_src.stage/v1" # as defined in native-apps-templates/basic + stage_name = "app_src.stage/v1" stage_files = runner.invoke_with_connection_json( ["stage", "list-files", f"{package_name}.{stage_name}"] ) @@ -274,17 +280,17 @@ def test_nativeapp_deploy_prune_w_stage_subdir( command, contains, not_contains, - nativeapp_project_directory, + nativeapp_teardown, runner, snapshot, print_paths_as_posix, default_username, resource_suffix, sanitize_deploy_output, + setup_v2_project_w_subdir, ): - test_project = "napp_stage_subdirs" - project_name = "stage_w_subdirs_pkg" - with nativeapp_project_directory(test_project): + project_name, _ = setup_v2_project_w_subdir() + with nativeapp_teardown(): result = runner.invoke_with_connection_json( ["app", "deploy", "--package-entity-id=pkg_v1"] ) @@ -299,7 +305,7 @@ def test_nativeapp_deploy_prune_w_stage_subdir( assert sanitize_deploy_output(result.output) == snapshot # verify the file does not exist on the stage - package_name = f"{project_name}_{default_username}{resource_suffix}".upper() + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() stage_name = "app_src.stage/v1" # as defined in native-apps-templates/basic stage_files = runner.invoke_with_connection_json( ["stage", "list-files", f"{package_name}.{stage_name}"] @@ -355,16 +361,17 @@ def test_nativeapp_deploy_files( @pytest.mark.integration def test_nativeapp_deploy_files_w_stage_subdir( - nativeapp_project_directory, + nativeapp_teardown, runner, snapshot, print_paths_as_posix, default_username, resource_suffix, sanitize_deploy_output, + setup_v2_project_w_subdir, ): - project_name = "stage_w_subdirs_pkg" - with nativeapp_project_directory("napp_stage_subdirs"): + project_name, _ = setup_v2_project_w_subdir() + with nativeapp_teardown(): # sync only two specific files to stage touch("app/v2/file.txt") result = runner.invoke_with_connection( @@ -379,7 +386,7 @@ def test_nativeapp_deploy_files_w_stage_subdir( assert sanitize_deploy_output(result.output) == snapshot # manifest and script files exist, readme doesn't exist - package_name = f"{project_name}_{default_username}{resource_suffix}".upper() + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() stage_name = "app_src.stage/v2" # as defined in native-apps-templates/basic stage_files = runner.invoke_with_connection_json( ["stage", "list-files", f"{package_name}.{stage_name}"] diff --git a/tests_integration/nativeapp/test_version.py b/tests_integration/nativeapp/test_version.py index 12bf881d40..fc272fb9f2 100644 --- a/tests_integration/nativeapp/test_version.py +++ b/tests_integration/nativeapp/test_version.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from shlex import split +from textwrap import dedent from typing import Any, Union import yaml @@ -102,13 +103,6 @@ def temporary_role(request, snowflake_session, resource_suffix): "ws version drop --entity-id=pkg", "napp_init_v2", ], - [ - "stage_w_subdirs", - "app version create --package-entity-id=pkg_v1", - "app version list --package-entity-id=pkg_v1", - "app version drop --package-entity-id=pkg_v1", - "napp_stage_subdirs", - ], ], ) def test_nativeapp_version_create_and_drop( @@ -155,6 +149,51 @@ def test_nativeapp_version_create_and_drop( assert len(actual.json) == 0 +@pytest.mark.integration +def test_nativeapp_version_create_and_drop_stage_subdir( + runner, + snowflake_session, + default_username, + resource_suffix, + nativeapp_teardown, + setup_v2_project_w_subdir, +): + project_name, _ = setup_v2_project_w_subdir() + drop_command = "app version drop --package-entity-id=pkg_v1" + list_command = "app version list --package-entity-id=pkg_v1" + create_command = "app version create --package-entity-id=pkg_v1" + with nativeapp_teardown(): + result_create = runner.invoke_with_connection_json( + [*split(create_command), "v1", "--force", "--skip-git-check"] + ) + assert result_create.exit_code == 0 + + # package exist + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() + assert contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"show application packages like '{package_name}'", + ) + ), + dict(name=package_name), + ) + + # app package contains version v1 + expect = snowflake_session.execute_string( + f"show versions in application package {package_name}" + ) + actual = runner.invoke_with_connection_json(split(list_command)) + assert actual.json == row_from_snowflake_session(expect) + + result_drop = runner.invoke_with_connection_json( + [*split(drop_command), "v1", "--force"] + ) + assert result_drop.exit_code == 0 + actual = runner.invoke_with_connection_json(split(list_command)) + assert len(actual.json) == 0 + + # Tests upgrading an app from an existing loose files installation to versioned installation. @pytest.mark.integration @pytest.mark.parametrize( @@ -207,15 +246,16 @@ def test_nativeapp_upgrade_w_subdirs( snowflake_session, default_username, resource_suffix, - nativeapp_project_directory, + nativeapp_teardown, + setup_v2_project_w_subdir, ): - project_name = "stage_w_subdirs" + project_name, _ = setup_v2_project_w_subdir() create_command = ( "app version create v1 --package-entity-id=pkg_v1 --force --skip-git-check" ) list_command = "app version list --package-entity-id=pkg_v1" drop_command = "app version drop --package-entity-id=pkg_v1" - with nativeapp_project_directory("napp_stage_subdirs"): + with nativeapp_teardown(): runner.invoke_with_connection_json(["app", "run", "--app-entity-id=app_v1"]) runner.invoke_with_connection_json(split(create_command)) diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md deleted file mode 100644 index 3107d92de2..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -This is the <% ctx.pkg_v1.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml deleted file mode 100644 index 3fa13420db..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ -# This is the v2 version of the napp_init_v1 project - -manifest_version: 1 - -version: - name: v1 - -artifacts: - setup_script: setup_script.sql - readme: README.md - extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql deleted file mode 100644 index 18017841cc..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/app/v1/setup_script.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE APPLICATION ROLE IF NOT EXISTS app_public; - -CREATE OR ALTER VERSIONED SCHEMA core; -GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md deleted file mode 100644 index b6162eae29..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -This is the <% ctx.pkg_v2.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml deleted file mode 100644 index ee1f6c99c5..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ -# This is the v2 version of the napp_init_v1 project - -manifest_version: 1 - -version: - name: v2 - -artifacts: - setup_script: setup_script.sql - readme: README.md - extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql deleted file mode 100644 index 18017841cc..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/app/v2/setup_script.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE APPLICATION ROLE IF NOT EXISTS app_public; - -CREATE OR ALTER VERSIONED SCHEMA core; -GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml b/tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml deleted file mode 100644 index c93cab9704..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs/snowflake.yml +++ /dev/null @@ -1,31 +0,0 @@ -definition_version: 2 -entities: - pkg_v1: - type: application package - identifier: <% fn.concat_ids('stage_w_subdirs_pkg_', ctx.env.USER) %> - stage_subdirectory: v1 - manifest: "" - artifacts: - - src: app/v1/* - dest: ./ - - pkg_v2: - type: application package - identifier: <% fn.concat_ids('stage_w_subdirs_pkg_', ctx.env.USER) %> - stage_subdirectory: v2 - manifest: "" - artifacts: - - src: app/v2/* - dest: ./ - - app_v1: - type: application - from: - target: pkg_v1 - identifier: <% fn.concat_ids('stage_w_subdirs_app_v1_', ctx.env.USER) %> - - app_v2: - type: application - from: - target: pkg_v2 - identifier: <% fn.concat_ids('stage_w_subdirs_app_v2_', ctx.env.USER) %> diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md deleted file mode 100644 index 3107d92de2..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -This is the <% ctx.pkg_v1.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml deleted file mode 100644 index 3fa13420db..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ -# This is the v2 version of the napp_init_v1 project - -manifest_version: 1 - -version: - name: v1 - -artifacts: - setup_script: setup_script.sql - readme: README.md - extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py deleted file mode 100644 index 335b485dd3..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/module-echo-v1/echo-v1.py +++ /dev/null @@ -1,13 +0,0 @@ -# This is where you can create python functions, which can further -# be used to create Snowpark UDFs and Stored Procedures in your setup_script.sql file. - -from snowflake.snowpark.functions import udf - -# UDF example: -# decorated example -@udf( - name="echo_fn", - native_app_params={"schema": "core", "application_roles": ["app_public"]}, -) -def echo_fn(data: str) -> str: - return "echo_fn, v1 implementation: " + data diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql deleted file mode 100644 index 18017841cc..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v1/setup_script.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE APPLICATION ROLE IF NOT EXISTS app_public; - -CREATE OR ALTER VERSIONED SCHEMA core; -GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md deleted file mode 100644 index b6162eae29..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -This is the <% ctx.pkg_v2.stage_subdirectory %> version of this package! diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml deleted file mode 100644 index ee1f6c99c5..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ -# This is the v2 version of the napp_init_v1 project - -manifest_version: 1 - -version: - name: v2 - -artifacts: - setup_script: setup_script.sql - readme: README.md - extension_code: true diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py deleted file mode 100644 index 6ae32cf48a..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/module-echo-v2/echo-v2.py +++ /dev/null @@ -1,13 +0,0 @@ -# This is where you can create python functions, which can further -# be used to create Snowpark UDFs and Stored Procedures in your setup_script.sql file. - -from snowflake.snowpark.functions import udf - -# UDF example: -# decorated example -@udf( - name="echo_fn", - native_app_params={"schema": "core", "application_roles": ["app_public"]}, -) -def echo_fn(data: str) -> str: - return "echo_fn, v2 implementation: " + data diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql deleted file mode 100644 index 18017841cc..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/app/v2/setup_script.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE APPLICATION ROLE IF NOT EXISTS app_public; - -CREATE OR ALTER VERSIONED SCHEMA core; -GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; diff --git a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml b/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml deleted file mode 100644 index 1f18f52062..0000000000 --- a/tests_integration/test_data/projects/napp_stage_subdirs_w_snowpark/snowflake.yml +++ /dev/null @@ -1,38 +0,0 @@ -definition_version: 2 -entities: - pkg_v1: - type: application package - identifier: <% fn.concat_ids('stage_w_subdirs_pkg', ctx.env.suffix) %> - stage_subdirectory: v1 - manifest: "" - artifacts: - - src: app/v1/* - dest: ./ - processors: - - snowpark - - pkg_v2: - type: application package - identifier: <% fn.concat_ids('stage_w_subdirs_pkg', ctx.env.suffix) %> - stage_subdirectory: v2 - manifest: "" - artifacts: - - src: app/v2/* - dest: ./ - processors: - - snowpark - - app_v1: - type: application - from: - target: pkg_v1 - identifier: <% fn.concat_ids('stage_w_subdirs_app_v1', ctx.env.suffix) %> - - app_v2: - type: application - from: - target: pkg_v2 - identifier: <% fn.concat_ids('stage_w_subdirs_app_v2', ctx.env.suffix) %> - -env: - suffix: <% fn.concat_ids('_', fn.sanitize_id(fn.get_username('unknown_user')) | lower) %> diff --git a/tests_integration/testing_utils/project_fixtures.py b/tests_integration/testing_utils/project_fixtures.py new file mode 100644 index 0000000000..16bf5e0653 --- /dev/null +++ b/tests_integration/testing_utils/project_fixtures.py @@ -0,0 +1,137 @@ +from textwrap import dedent + +import pytest + +from tests.nativeapp.factories import ( + ProjectV2Factory, + ApplicationPackageEntityModelFactory, + ApplicationEntityModelFactory, +) + + +MANIFEST_BASIC = dedent( + """\ + manifest_version: 1 + + version: + name: dev + + artifacts: + setup_script: setup_script.sql + readme: README.md + extension_code: true + """ +) + +PYTHON_W_SNOWPARK = dedent( + """\ + from snowflake.snowpark.functions import udf + @udf( + name="echo_fn", + native_app_params={"schema": "core", "application_roles": ["app_public"]}, + ) + def echo_fn(data: str) -> str: + return "echo_fn: " + data + """ +) + + +@pytest.fixture +def setup_v2_project_w_subdir(temp_dir): + def wrapper(): + readme_v1 = ( + "This is the <% ctx.pkg_v1.stage_subdirectory %> version of this package!" + ) + readme_v2 = ( + "This is the <% ctx.pkg_v2.stage_subdirectory %> version of this package!" + ) + project_name = "stage_w_subdirs" + ProjectV2Factory( + pdf__entities=dict( + pkg_v1=ApplicationPackageEntityModelFactory( + identifier=f"<% fn.concat_ids('{project_name}_pkg_', ctx.env.USER) %>", + manifest="", + artifacts=[{"src": "app/v1/*", "dest": "./"}], + stage_subdirectory="v1", + ), + app_v1=ApplicationEntityModelFactory( + fromm__target="pkg_v1", + identifier=f"<% fn.concat_ids('{project_name}_app_v1_', ctx.env.USER) %>", + ), + pkg_v2=ApplicationPackageEntityModelFactory( + identifier=f"<% fn.concat_ids('{project_name}_pkg_', ctx.env.USER) %>", + manifest="", + artifacts=[{"src": "app/v2/*", "dest": "./"}], + stage_subdirectory="v2", + ), + app_v2=ApplicationEntityModelFactory( + fromm__target="pkg_v2", + identifier=f"<% fn.concat_ids('{project_name}_app_v2_', ctx.env.USER) %>", + ), + ), + files={ + "app/v1/manifest.yml": MANIFEST_BASIC, + "app/v1/README.md": readme_v1, + "app/v1/setup_script.sql": "SELECT 1;", + "app/v2/manifest.yml": MANIFEST_BASIC, + "app/v2/README.md": readme_v2, + "app/v2/setup_script.sql": "SELECT 1;", + }, + ) + return project_name, temp_dir + + return wrapper + + +@pytest.fixture +def setup_v2_project_w_subdir_w_snowpark(temp_dir): + def wrapper(): + setup_script = dedent( + """\ + CREATE APPLICATION ROLE IF NOT EXISTS app_public; + CREATE OR ALTER VERSIONED SCHEMA core; + GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; + """ + ) + project_name = "stage_w_subdirs" + ProjectV2Factory( + pdf__entities=dict( + pkg_v1=ApplicationPackageEntityModelFactory( + identifier=f"<% fn.concat_ids('{project_name}_pkg_', ctx.env.USER) %>", + manifest="", + artifacts=[ + {"src": "app/v1/*", "dest": "./", "processors": ["snowpark"]} + ], + stage_subdirectory="v1", + ), + app_v1=ApplicationEntityModelFactory( + fromm__target="pkg_v1", + identifier=f"<% fn.concat_ids('{project_name}_app_v1_', ctx.env.USER) %>", + ), + pkg_v2=ApplicationPackageEntityModelFactory( + identifier=f"<% fn.concat_ids('{project_name}_pkg_', ctx.env.USER) %>", + manifest="", + artifacts=[ + {"src": "app/v2/*", "dest": "./", "processors": ["snowpark"]} + ], + stage_subdirectory="v2", + ), + app_v2=ApplicationEntityModelFactory( + fromm__target="pkg_v2", + identifier=f"<% fn.concat_ids('{project_name}_app_v2_', ctx.env.USER) %>", + ), + ), + files={ + "app/v1/manifest.yml": MANIFEST_BASIC, + "app/v1/README.md": "\n", + "app/v1/setup_script.sql": setup_script, + "app/v1/module-echo-v1/echo-v1.py": PYTHON_W_SNOWPARK, + "app/v2/manifest.yml": MANIFEST_BASIC, + "app/v2/README.md": "\n", + "app/v2/setup_script.sql": setup_script, + "app/v2/module-echo-v2/echo-v2.py": PYTHON_W_SNOWPARK, + }, + ) + return project_name, temp_dir + + return wrapper From a225d44de64c82e2596dca84aa7f5d737c5ce6b5 Mon Sep 17 00:00:00 2001 From: PARYA JAFARI Date: Tue, 17 Dec 2024 11:54:39 -0500 Subject: [PATCH 33/33] import fixture --- tests_integration/nativeapp/test_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_integration/nativeapp/test_version.py b/tests_integration/nativeapp/test_version.py index fc272fb9f2..26110191bd 100644 --- a/tests_integration/nativeapp/test_version.py +++ b/tests_integration/nativeapp/test_version.py @@ -30,7 +30,7 @@ ) from tests.project.fixtures import * from tests_integration.test_utils import contains_row_with, row_from_snowflake_session - +from tests_integration.testing_utils.project_fixtures import * # A minimal set of fields to compare when checking version output VERSION_FIELDS_TO_OUTPUT = [