Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add optional subdirectory for stage in application package #1916

Open
wants to merge 41 commits into
base: main
Choose a base branch
from
Open
Changes from 27 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
8617252
teardown with comments
sfc-gh-pjafari Nov 29, 2024
5a3e154
make version work
sfc-gh-pjafari Nov 29, 2024
8db7a40
update path for version create
sfc-gh-pjafari Dec 2, 2024
d9eb5f4
catch tempalte processor file read error
sfc-gh-pjafari Dec 2, 2024
71108d0
make diff an entity action
sfc-gh-pjafari Dec 2, 2024
a7e3a20
remove teardown with all apps logic
sfc-gh-pjafari Dec 2, 2024
99abebf
ws change
sfc-gh-pjafari Dec 2, 2024
ce71ebf
add schema to StagePathParts
sfc-gh-pjafari Dec 3, 2024
855b413
use DefaultStagePathParts for stage in NA
sfc-gh-pjafari Dec 3, 2024
68904aa
remmove comments
sfc-gh-pjafari Dec 3, 2024
5fc642e
clean up
sfc-gh-pjafari Dec 3, 2024
49275b9
clean up
sfc-gh-pjafari Dec 3, 2024
04d807e
update comment
sfc-gh-pjafari Dec 3, 2024
be0402b
merge main
sfc-gh-pjafari Dec 3, 2024
ff369e7
rename stage
sfc-gh-pjafari Dec 3, 2024
31e1f70
remove trailing slash when no directory
sfc-gh-pjafari Dec 3, 2024
f52c6cb
update unit tests
sfc-gh-pjafari Dec 3, 2024
e95ab86
update docstring
sfc-gh-pjafari Dec 3, 2024
b663798
add unit tests for DefaultStagePathParts
sfc-gh-pjafari Dec 4, 2024
3412b3c
Merge branch 'pj-schema-in-stagepathparts' into pj-subartifacts-subdirs
sfc-gh-pjafari Dec 4, 2024
08a4110
update stage diff command and message wording
sfc-gh-pjafari Dec 4, 2024
c7a0a88
update snapshot for test_deploy
sfc-gh-pjafari Dec 4, 2024
8153807
Merge branch 'main' into pj-subartifacts-subdirs
sfc-gh-pjafari Dec 4, 2024
504f692
add integration tests
sfc-gh-pjafari Dec 4, 2024
5cbf941
Merge branch 'main' into pj-subartifacts-subdirs
sfc-gh-pjafari Dec 4, 2024
56fc4f6
iMerge branch 'pj-subartifacts-subdirs' of https://github.com/snowfla…
sfc-gh-pjafari Dec 4, 2024
b81670c
add unit tests
sfc-gh-pjafari Dec 5, 2024
889c77e
update release notes
sfc-gh-pjafari Dec 5, 2024
44263ce
remove template processor change
sfc-gh-pjafari Dec 5, 2024
70e3d38
make stage paths work for quoted ids
sfc-gh-pjafari Dec 6, 2024
c0b0826
address comments
sfc-gh-pjafari Dec 6, 2024
4ec1071
make for quoted identifiers in diff
sfc-gh-pjafari Dec 7, 2024
00db5a8
merge main
sfc-gh-pjafari Dec 12, 2024
7f30fea
make diff work with all identifiers, update docstrings
sfc-gh-pjafari Dec 12, 2024
1921805
update tests
sfc-gh-pjafari Dec 12, 2024
3f9ed94
merge main
sfc-gh-pjafari Dec 13, 2024
d84a59c
fix tests
sfc-gh-pjafari Dec 13, 2024
d865ac8
update commands
sfc-gh-pjafari Dec 17, 2024
dfb4373
merge main
sfc-gh-pjafari Dec 17, 2024
aa80ac4
replace test data with factory
sfc-gh-pjafari Dec 17, 2024
a225d44
import fixture
sfc-gh-pjafari Dec 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -58,38 +58,45 @@ def expand_templates_in_file(
if src.is_dir():
return

with self.edit_file(dest) as file:
if not has_client_side_templates(file.contents) and not (
_is_sql_file(dest) and has_sql_templates(file.contents)
):
return

src_file_name = src.relative_to(self._bundle_ctx.project_root)
cc.step(f"Expanding templates in {src_file_name}")
with cc.indented():
try:
jinja_env = (
choose_sql_jinja_env_based_on_template_syntax(
file.contents, reference_name=src_file_name
src_file_name = src.relative_to(self._bundle_ctx.project_root)

try:
with self.edit_file(dest) as file:
if not has_client_side_templates(file.contents) and not (
_is_sql_file(dest) and has_sql_templates(file.contents)
):
return
cc.step(f"Expanding templates in {src_file_name}")
with cc.indented():
try:
jinja_env = (
choose_sql_jinja_env_based_on_template_syntax(
file.contents, reference_name=src_file_name
)
if _is_sql_file(dest)
else get_client_side_jinja_env()
)
if _is_sql_file(dest)
else get_client_side_jinja_env()
)
expanded_template = jinja_env.from_string(file.contents).render(
template_context or get_cli_context().template_context
)

# For now, we are printing the source file path in the error message
# instead of the destination file path to make it easier for the user
# to identify the file that has the error, and edit the correct file.
except jinja2.TemplateSyntaxError as e:
raise InvalidTemplateInFileError(src_file_name, e, e.lineno) from e

except jinja2.UndefinedError as e:
raise InvalidTemplateInFileError(src_file_name, e) from e

if expanded_template != file.contents:
file.edited_contents = expanded_template
expanded_template = jinja_env.from_string(file.contents).render(
template_context or get_cli_context().template_context
)

# For now, we are printing the source file path in the error message
# instead of the destination file path to make it easier for the user
# to identify the file that has the error, and edit the correct file.
except jinja2.TemplateSyntaxError as e:
raise InvalidTemplateInFileError(
src_file_name, e, e.lineno
) from e

except jinja2.UndefinedError as e:
raise InvalidTemplateInFileError(src_file_name, e) from e

if expanded_template != file.contents:
file.edited_contents = expanded_template
except UnicodeDecodeError as err:
cc.warning(
f"Could not read file {src_file_name}, error: {err.reason}. Skipping this file."
)
sfc-gh-pjafari marked this conversation as resolved.
Show resolved Hide resolved

@span("templates_processor")
def process(
33 changes: 17 additions & 16 deletions src/snowflake/cli/_plugins/nativeapp/commands.py
Original file line number Diff line number Diff line change
@@ -37,11 +37,6 @@
force_project_definition_v2,
)
from snowflake.cli._plugins.nativeapp.version.commands import app as versions_app
from snowflake.cli._plugins.stage.diff import (
DiffResult,
compute_stage_diff,
)
from snowflake.cli._plugins.stage.utils import print_diff_to_console
from snowflake.cli._plugins.workspace.manager import WorkspaceManager
from snowflake.cli.api.cli_global_context import get_cli_context
from snowflake.cli.api.commands.decorators import (
@@ -109,20 +104,15 @@ def app_diff(
project_root=cli_context.project_root,
)
package_id = options["package_entity_id"]
package = cli_context.project_definition.entities[package_id]
bundle_map = ws.perform_action(
diff = ws.perform_action(
package_id,
EntityActions.BUNDLE,
)
stage_fqn = f"{package.fqn.name}.{package.stage}"
diff: DiffResult = compute_stage_diff(
local_root=Path(package.deploy_root), stage_fqn=stage_fqn
EntityActions.DIFF,
print_to_console=cli_context.output_format != OutputFormat.JSON,
)
if cli_context.output_format == OutputFormat.JSON:
return ObjectResult(diff.to_dict())
else:
print_diff_to_console(diff, bundle_map)
return None # don't print any output

return None
Comment on lines +116 to +124
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Made diff its own entity action. action_diff



@app.command("run", requires_connection=True)
@@ -247,11 +237,22 @@ def app_teardown(
project_definition=cli_context.project_definition,
project_root=cli_context.project_root,
)

# TODO: get all apps created from this application package from snowflake, compare, confirm and drop.
# TODO: add messaging/confirmation here for extra apps found as part of above
Comment on lines +257 to +258
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Kept current teardown functionality. Next PR, we will get and drop all app objects created from this package, not just what's listed in pdf.

all_packages_with_id = [
package_entity.entity_id
for package_entity in project.get_entities_by_type(
ApplicationPackageEntityModel.get_type()
).values()
if package_entity.identifier == app_package_entity.identifier
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this work with identifiers (quoted/unquoted)? or is this == overridden to do that, and if so, identifier could be String or Identifier.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

alternatively, you can take a look into package_entity.fqn

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point, replaced with same_identifier

]

for app_entity in project.get_entities_by_type(
ApplicationEntityModel.get_type()
).values():
# Drop each app
if app_entity.from_.target == app_package_entity.entity_id:
if app_entity.from_.target in all_packages_with_id:
ws.perform_action(
app_entity.entity_id,
EntityActions.DROP,
33 changes: 15 additions & 18 deletions src/snowflake/cli/_plugins/nativeapp/entities/application.py
Original file line number Diff line number Diff line change
@@ -53,6 +53,7 @@
UpgradeApplicationRestrictionError,
)
from snowflake.cli._plugins.nativeapp.utils import needs_confirmation
from snowflake.cli._plugins.stage.manager import DefaultStagePathParts
from snowflake.cli._plugins.workspace.context import ActionContext
from snowflake.cli.api.cli_global_context import get_cli_context, span
from snowflake.cli.api.console.abc import AbstractConsole
@@ -83,7 +84,6 @@
from snowflake.cli.api.project.schemas.updatable_model import DiscriminatorField
from snowflake.cli.api.project.util import (
append_test_resource_suffix,
extract_schema,
identifier_for_url,
to_identifier,
unquote_identifier,
@@ -330,7 +330,6 @@ def action_deploy(
recursive: bool,
paths: List[Path],
validate: bool = ValidateOption,
stage_fqn: Optional[str] = None,
interactive: bool = InteractiveOption,
version: Optional[str] = None,
patch: Optional[int] = None,
@@ -345,7 +344,8 @@ def action_deploy(
package_entity: ApplicationPackageEntity = action_ctx.get_entity(
self.package_entity_id
)
stage_fqn = stage_fqn or package_entity.stage_fqn

stage_path = package_entity.stage_path

if force:
policy = AllowAlwaysPolicy()
@@ -358,7 +358,7 @@ def action_deploy(
if from_release_directive:
self.create_or_upgrade_app(
package=package_entity,
stage_fqn=stage_fqn,
stage_path=stage_path,
install_method=SameAccountInstallMethod.release_directive(),
policy=policy,
interactive=interactive,
@@ -380,7 +380,7 @@ def action_deploy(

self.create_or_upgrade_app(
package=package_entity,
stage_fqn=stage_fqn,
stage_path=stage_path,
install_method=SameAccountInstallMethod.versioned_dev(version, patch),
policy=policy,
interactive=interactive,
@@ -394,13 +394,12 @@ def action_deploy(
recursive=True,
paths=[],
validate=validate,
stage_fqn=stage_fqn,
interactive=interactive,
force=force,
)
self.create_or_upgrade_app(
package=package_entity,
stage_fqn=stage_fqn,
stage_path=stage_path,
install_method=SameAccountInstallMethod.unversioned_dev(),
policy=policy,
interactive=interactive,
@@ -598,7 +597,7 @@ def get_objects_owned_by_application(self) -> List[ApplicationOwnedObject]:

def _upgrade_app(
self,
stage_fqn: str,
stage_path: DefaultStagePathParts,
install_method: SameAccountInstallMethod,
event_sharing: EventSharingHandler,
policy: PolicyBase,
@@ -610,7 +609,7 @@ def _upgrade_app(
return get_snowflake_facade().upgrade_application(
name=self.name,
install_method=install_method,
stage_fqn=stage_fqn,
stage_path_to_artifacts=stage_path.full_path,
debug_mode=self.debug,
should_authorize_event_sharing=event_sharing.should_authorize_event_sharing(),
role=self.role,
@@ -623,7 +622,7 @@ def _upgrade_app(

def _create_app(
self,
stage_fqn: str,
stage_path: DefaultStagePathParts,
install_method: SameAccountInstallMethod,
event_sharing: EventSharingHandler,
package: ApplicationPackageEntity,
@@ -639,7 +638,7 @@ def _create_app(
role_to_use=package.role,
)

stage_schema = extract_schema(stage_fqn)
stage_schema = stage_path.schema
get_snowflake_facade().grant_privileges_to_role(
privileges=["usage"],
object_type=ObjectType.SCHEMA,
@@ -651,7 +650,7 @@ def _create_app(
get_snowflake_facade().grant_privileges_to_role(
privileges=["read"],
object_type=ObjectType.STAGE,
object_identifier=stage_fqn,
object_identifier=stage_path.stage,
role_to_grant=self.role,
role_to_use=package.role,
)
@@ -660,7 +659,7 @@ def _create_app(
name=self.name,
package_name=package.name,
install_method=install_method,
stage_fqn=stage_fqn,
stage_path_to_artifacts=stage_path.full_path,
debug_mode=self.debug,
should_authorize_event_sharing=event_sharing.should_authorize_event_sharing(),
role=self.role,
@@ -671,7 +670,7 @@ def _create_app(
def create_or_upgrade_app(
self,
package: ApplicationPackageEntity,
stage_fqn: str,
stage_path: DefaultStagePathParts,
install_method: SameAccountInstallMethod,
policy: PolicyBase,
interactive: bool,
@@ -688,13 +687,11 @@ def create_or_upgrade_app(
self.name, self.role
)

stage_fqn = stage_fqn or package.stage_fqn

# 2. If existing application is found, try to upgrade the application object.
create_or_upgrade_result = None
if show_app_row:
create_or_upgrade_result = self._upgrade_app(
stage_fqn=stage_fqn,
stage_path=stage_path,
install_method=install_method,
event_sharing=event_sharing,
policy=policy,
@@ -704,7 +701,7 @@ def create_or_upgrade_app(
# 3. If no existing application found, or we performed a drop before the upgrade, we proceed to create
if create_or_upgrade_result is None:
create_or_upgrade_result = self._create_app(
stage_fqn=stage_fqn,
stage_path=stage_path,
install_method=install_method,
event_sharing=event_sharing,
package=package,
Loading
Loading