Skip to content

Commit

Permalink
POC: Add child entities to application package (#1856)
Browse files Browse the repository at this point in the history 
* add child entities

* children_artifacts_dir

* unit tests

* sanitize dir name docstring

* error message on child directory collision
  • Loading branch information
@sfc-gh-gbloom
sfc-gh-gbloom authored Dec 12, 2024
1 parent 9305955 commit 5a3e36e
Show file tree
Hide file tree
Showing 19 changed files with 598 additions and 30 deletions.
5 changes: 4 additions & 1 deletion src/snowflake/cli/_plugins/nativeapp/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,10 @@ def app_validate(
if cli_context.output_format == OutputFormat.JSON:
return ObjectResult(
package.get_validation_result(
use_scratch_stage=True, interactive=False, force=True
action_ctx=ws.action_ctx,
use_scratch_stage=True,
interactive=False,
force=True,
)
)

Expand Down
181 changes: 168 additions & 13 deletions src/snowflake/cli/_plugins/nativeapp/entities/application_package.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
from __future__ import annotations

import json
import os
import re
from pathlib import Path
from textwrap import dedent
from typing import Any, List, Literal, Optional, Union
from typing import Any, List, Literal, Optional, Set, Union

import typer
from click import BadOptionUsage, ClickException
Expand All @@ -14,6 +15,7 @@
BundleMap,
VersionInfo,
build_bundle,
find_setup_script_file,
find_version_info_in_manifest_file,
)
from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
Expand All @@ -30,6 +32,9 @@
PATCH_COL,
VERSION_COL,
)
from snowflake.cli._plugins.nativeapp.entities.application_package_child_interface import (
ApplicationPackageChildInterface,
)
from snowflake.cli._plugins.nativeapp.exceptions import (
ApplicationPackageAlreadyExistsError,
ApplicationPackageDoesNotExistError,
Expand All @@ -48,9 +53,16 @@
from snowflake.cli._plugins.nativeapp.sf_facade_exceptions import (
InsufficientPrivilegesError,
)
from snowflake.cli._plugins.nativeapp.utils import needs_confirmation
from snowflake.cli._plugins.nativeapp.utils import needs_confirmation, sanitize_dir_name
from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
FunctionEntityModel,
ProcedureEntityModel,
)
from snowflake.cli._plugins.stage.diff import DiffResult
from snowflake.cli._plugins.stage.manager import StageManager
from snowflake.cli._plugins.streamlit.streamlit_entity_model import (
StreamlitEntityModel,
)
from snowflake.cli._plugins.workspace.context import ActionContext
from snowflake.cli.api.cli_global_context import span
from snowflake.cli.api.entities.common import (
Expand All @@ -75,6 +87,7 @@
from snowflake.cli.api.project.schemas.updatable_model import (
DiscriminatorField,
IdentifierField,
UpdatableModel,
)
from snowflake.cli.api.project.schemas.v1.native_app.package import DistributionOptions
from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping
Expand All @@ -94,30 +107,71 @@
from snowflake.connector import DictCursor, ProgrammingError
from snowflake.connector.cursor import SnowflakeCursor

ApplicationPackageChildrenTypes = (
StreamlitEntityModel | FunctionEntityModel | ProcedureEntityModel
)


class ApplicationPackageChildIdentifier(UpdatableModel):
schema_: Optional[str] = Field(
title="Child entity schema", alias="schema", default=None
)


class EnsureUsableByField(UpdatableModel):
application_roles: Optional[Union[str, Set[str]]] = Field(
title="One or more application roles to be granted with the required privileges",
default=None,
)

@field_validator("application_roles")
@classmethod
def ensure_app_roles_is_a_set(
cls, application_roles: Optional[Union[str, Set[str]]]
) -> Optional[Union[Set[str]]]:
if isinstance(application_roles, str):
return set([application_roles])
return application_roles


class ApplicationPackageChildField(UpdatableModel):
target: str = Field(title="The key of the entity to include in this package")
ensure_usable_by: Optional[EnsureUsableByField] = Field(
title="Automatically grant the required privileges on the child object and its schema",
default=None,
)
identifier: ApplicationPackageChildIdentifier = Field(
title="Entity identifier", default=None
)


class ApplicationPackageEntityModel(EntityModelBase):
type: Literal["application package"] = DiscriminatorField() # noqa: A003
artifacts: List[Union[PathMapping, str]] = Field(
title="List of paths or file source/destination pairs to add to the deploy root",
)
bundle_root: Optional[str] = Field(
title="Folder at the root of your project where artifacts necessary to perform the bundle step are stored.",
title="Folder at the root of your project where artifacts necessary to perform the bundle step are stored",
default="output/bundle/",
)
deploy_root: Optional[str] = Field(
title="Folder at the root of your project where the build step copies the artifacts",
default="output/deploy/",
)
children_artifacts_dir: Optional[str] = Field(
title="Folder under deploy_root where the child artifacts will be stored",
default="_children/",
)
generated_root: Optional[str] = Field(
title="Subdirectory of the deploy root where files generated by the Snowflake CLI will be written.",
title="Subdirectory of the deploy root where files generated by the Snowflake CLI will be written",
default="__generated/",
)
stage: Optional[str] = IdentifierField(
title="Identifier of the stage that stores the application artifacts.",
title="Identifier of the stage that stores the application artifacts",
default="app_src.stage",
)
scratch_stage: Optional[str] = IdentifierField(
title="Identifier of the stage that stores temporary scratch data used by the Snowflake CLI.",
title="Identifier of the stage that stores temporary scratch data used by the Snowflake CLI",
default="app_src.stage_snowflake_cli_scratch",
)
distribution: Optional[DistributionOptions] = Field(
Expand All @@ -128,6 +182,19 @@ class ApplicationPackageEntityModel(EntityModelBase):
title="Path to manifest.yml. Unused and deprecated starting with Snowflake CLI 3.2",
default="",
)
children: Optional[List[ApplicationPackageChildField]] = Field(
title="Entities that will be bundled and deployed as part of this application package",
default=[],
)

@field_validator("children")
@classmethod
def verify_children_behind_flag(
cls, input_value: Optional[List[ApplicationPackageChildField]]
) -> Optional[List[ApplicationPackageChildField]]:
if input_value and not FeatureFlag.ENABLE_NATIVE_APP_CHILDREN.is_enabled():
raise AttributeError("Application package children are not supported yet")
return input_value

@field_validator("identifier")
@classmethod
Expand Down Expand Up @@ -183,6 +250,10 @@ def project_root(self) -> Path:
def deploy_root(self) -> Path:
return self.project_root / self._entity_model.deploy_root

@property
def children_artifacts_deploy_root(self) -> Path:
return self.deploy_root / self._entity_model.children_artifacts_dir

@property
def bundle_root(self) -> Path:
return self.project_root / self._entity_model.bundle_root
Expand Down Expand Up @@ -221,7 +292,7 @@ def post_deploy_hooks(self) -> list[PostDeployHook] | None:
return model.meta and model.meta.post_deploy

def action_bundle(self, action_ctx: ActionContext, *args, **kwargs):
return self._bundle()
return self._bundle(action_ctx)

def action_deploy(
self,
Expand All @@ -237,6 +308,7 @@ def action_deploy(
**kwargs,
):
return self._deploy(
action_ctx=action_ctx,
bundle_map=None,
prune=prune,
recursive=recursive,
Expand Down Expand Up @@ -336,6 +408,7 @@ def action_validate(
**kwargs,
):
self.validate_setup_script(
action_ctx=action_ctx,
use_scratch_stage=use_scratch_stage,
interactive=interactive,
force=force,
Expand Down Expand Up @@ -390,7 +463,7 @@ def action_version_create(
else:
git_policy = AllowAlwaysPolicy()

bundle_map = self._bundle()
bundle_map = self._bundle(action_ctx)
resolved_version, resolved_patch, resolved_label = self.resolve_version_info(
version=version,
patch=patch,
Expand All @@ -404,6 +477,7 @@ def action_version_create(
self.check_index_changes_in_git_repo(policy=policy, interactive=interactive)

self._deploy(
action_ctx=action_ctx,
bundle_map=bundle_map,
prune=True,
recursive=True,
Expand Down Expand Up @@ -507,7 +581,7 @@ def action_version_drop(
"""
)
)
self._bundle()
self._bundle(action_ctx)
version_info = find_version_info_in_manifest_file(self.deploy_root)
version = version_info.version_name
if not version:
Expand Down Expand Up @@ -692,7 +766,7 @@ def action_release_directive_unset(
role=self.role,
)

def _bundle(self):
def _bundle(self, action_ctx: ActionContext = None):
model = self._entity_model
bundle_map = build_bundle(self.project_root, self.deploy_root, model.artifacts)
bundle_context = BundleContext(
Expand All @@ -705,10 +779,80 @@ def _bundle(self):
)
compiler = NativeAppCompiler(bundle_context)
compiler.compile_artifacts()

if self._entity_model.children:
# Bundle children and append their SQL to setup script
# TODO Consider re-writing the logic below as a processor
children_sql = self._bundle_children(action_ctx=action_ctx)
setup_file_path = find_setup_script_file(deploy_root=self.deploy_root)
with open(setup_file_path, "r", encoding="utf-8") as file:
existing_setup_script = file.read()
if setup_file_path.is_symlink():
setup_file_path.unlink()
with open(setup_file_path, "w", encoding="utf-8") as file:
file.write(existing_setup_script)
file.write("\n-- AUTO GENERATED CHILDREN SECTION\n")
file.write("\n".join(children_sql))
file.write("\n")

return bundle_map

def _bundle_children(self, action_ctx: ActionContext) -> List[str]:
# Create _children directory
children_artifacts_dir = self.children_artifacts_deploy_root
os.makedirs(children_artifacts_dir)
children_sql = []
for child in self._entity_model.children:
# Create child sub directory
child_artifacts_dir = children_artifacts_dir / sanitize_dir_name(
child.target
)
try:
os.makedirs(child_artifacts_dir)
except FileExistsError:
raise ClickException(
f"Could not create sub-directory at {child_artifacts_dir}. Make sure child entity names do not collide with each other."
)
child_entity: ApplicationPackageChildInterface = action_ctx.get_entity(
child.target
)
child_entity.bundle(child_artifacts_dir)
app_role = (
to_identifier(
child.ensure_usable_by.application_roles.pop() # TODO Support more than one application role
)
if child.ensure_usable_by and child.ensure_usable_by.application_roles
else None
)
child_schema = (
to_identifier(child.identifier.schema_)
if child.identifier and child.identifier.schema_
else None
)
children_sql.append(
child_entity.get_deploy_sql(
artifacts_dir=child_artifacts_dir.relative_to(self.deploy_root),
schema=child_schema,
)
)
if app_role:
children_sql.append(
f"CREATE APPLICATION ROLE IF NOT EXISTS {app_role};"
)
if child_schema:
children_sql.append(
f"GRANT USAGE ON SCHEMA {child_schema} TO APPLICATION ROLE {app_role};"
)
children_sql.append(
child_entity.get_usage_grant_sql(
app_role=app_role, schema=child_schema
)
)
return children_sql

def _deploy(
self,
action_ctx: ActionContext,
bundle_map: BundleMap | None,
prune: bool,
recursive: bool,
Expand All @@ -733,7 +877,7 @@ def _deploy(
stage_fqn = stage_fqn or self.stage_fqn

# 1. Create a bundle if one wasn't passed in
bundle_map = bundle_map or self._bundle()
bundle_map = bundle_map or self._bundle(action_ctx)

# 2. Create an empty application package, if none exists
try:
Expand Down Expand Up @@ -765,6 +909,7 @@ def _deploy(

if validate:
self.validate_setup_script(
action_ctx=action_ctx,
use_scratch_stage=False,
interactive=interactive,
force=force,
Expand Down Expand Up @@ -1054,14 +1199,19 @@ def execute_post_deploy_hooks(self):
)

def validate_setup_script(
self, use_scratch_stage: bool, interactive: bool, force: bool
self,
action_ctx: ActionContext,
use_scratch_stage: bool,
interactive: bool,
force: bool,
):
workspace_ctx = self._workspace_ctx
console = workspace_ctx.console

"""Validates Native App setup script SQL."""
with console.phase(f"Validating Snowflake Native App setup script."):
validation_result = self.get_validation_result(
action_ctx=action_ctx,
use_scratch_stage=use_scratch_stage,
force=force,
interactive=interactive,
Expand All @@ -1083,13 +1233,18 @@ def validate_setup_script(

@span("validate_setup_script")
def get_validation_result(
self, use_scratch_stage: bool, interactive: bool, force: bool
self,
action_ctx: ActionContext,
use_scratch_stage: bool,
interactive: bool,
force: bool,
):
"""Call system$validate_native_app_setup() to validate deployed Native App setup script."""
stage_fqn = self.stage_fqn
if use_scratch_stage:
stage_fqn = self.scratch_stage_fqn
self._deploy(
action_ctx=action_ctx,
bundle_map=None,
prune=True,
recursive=True,
Expand Down
Loading

0 comments on commit 5a3e36e

Please sign in to comment.