From 0d71bd3b37ee3b75b7cb0e34e5212e0f8dbe4933 Mon Sep 17 00:00:00 2001 From: Cam Gorrie Date: Tue, 12 Dec 2023 14:57:51 -0500 Subject: [PATCH] more scaffolding --- src/snowcli/cli/appify/commands.py | 11 +++++-- src/snowcli/cli/appify/generate.py | 50 ++++++++++++++++++++++++++---- src/snowcli/cli/appify/util.py | 18 ++++++++--- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/src/snowcli/cli/appify/commands.py b/src/snowcli/cli/appify/commands.py index 5938628c82..2d2b33ebfa 100644 --- a/src/snowcli/cli/appify/commands.py +++ b/src/snowcli/cli/appify/commands.py @@ -9,7 +9,7 @@ from snowcli.output.types import CommandResult, MessageResult from snowcli.cli.appify.metadata import MetadataDumper -from snowcli.cli.appify.generate import modify_yml +from snowcli.cli.appify.generate import modifications, generate_setup_statements from strictyaml import as_document @@ -46,8 +46,15 @@ def appify( dumper = MetadataDumper(db, project.path) dumper.execute() + # catalog = load_catalog(dumper.catalog_path) + # TODO: post-process: re-write stage references in-place + # TODO: setup_statements = list(generate_setup_statements(catalog)) + # with open(project.path / "app" / "setup.sql") as setup_sql: + # setup_sql.write(";\n".join(setup_statements)) + # setup_sql.write(";") + # include referenced stages + metadata in our app stage - with modify_yml(project.path / "snowflake.yml") as snowflake_yml: + with modifications(project.path / "snowflake.yml") as snowflake_yml: artifacts = snowflake_yml["native_app"]["artifacts"].data artifacts.append(str(dumper.metadata_path)) artifacts.append(str(dumper.stages_path)) diff --git a/src/snowcli/cli/appify/generate.py b/src/snowcli/cli/appify/generate.py index 128f9c20ba..e09e7b2acd 100644 --- a/src/snowcli/cli/appify/generate.py +++ b/src/snowcli/cli/appify/generate.py @@ -1,16 +1,28 @@ from typing import Generator, List, Tuple + +import json from contextlib import contextmanager from pathlib import Path from strictyaml import YAML, load +from snowcli.cli.appify.util import split_schema_and_object_id + from snowcli.cli.project.util import to_identifier from snowcli.cli.project.schemas.project_definition import project_schema APP_PUBLIC = "app_public" +CALLABLE_KINDS = ["function", "procedure"] +GRANT_BY_KIND = { + "function": "usage on function", + "procedure": "usage on procedure", + "table": "select on table", # FIXME: what if they want editing? what if they want nothing? + "view": "select on view", + "streamlit": "usage on streamlit", +} @contextmanager -def modify_yml(path: Path) -> Generator[YAML, None, None]: +def modifications(path: Path) -> Generator[YAML, None, None]: """ Read, then write back modifications made to a project definition file. """ @@ -23,25 +35,51 @@ def modify_yml(path: Path) -> Generator[YAML, None, None]: f.write(yml.as_yaml()) -def get_ordering(reference_json: Path) -> List[Tuple[str, str]]: +def get_ordering(catalog: dict) -> List[Tuple[str, str]]: """ Return a list of (schema, object name) tuples that represent a depth-first search of the DAG that represents their dependencies. + Object names must include arguments for callable types. """ return [] +def get_kind(catalog: dict, schema: str, object_name: str) -> str: + """ + Determine the kind of an object based on the metadata catalog. + """ + pass + + +def load_catalog(catalog_json: Path) -> dict: + """ + Returns the metadata catalog for the database, containing reference + and kind information of the objects we dumped metadata for. + """ + with open(catalog_json, "r") as f: + return json.load(f) + + def generate_setup_statements( - stages_path: Path, - metadata_path: Path, - reference_json: Path, + catalog: dict, ) -> Generator[str, None, None]: """ Generator that yields all the statements necessary to build the setup script. """ yield f"create application role if not exists {APP_PUBLIC}" - schemas = [f.name for f in sorted(metadata_path.iterdir()) if f.is_dir()] + all_object_ids = list(catalog.keys()) + schemas = list(set([split_schema_and_object_id(x)[0] for x in all_object_ids])) + for schema in schemas: yield f"create or alter versioned schema {to_identifier(schema)}" yield f"grant usage on schema {to_identifier(schema)} to application role {APP_PUBLIC}" + + for schema, object_name in get_ordering(catalog): + kind = get_kind(catalog, schema, object_name) + yield f"use schema {to_identifier(schema)}" + # XXX: is this correct quoting? + yield f"execute immediate from './metadata/{schema}/{object_name}.sql'" + if kind in GRANT_BY_KIND: + # FIXME: need to refactor to split name + arguments so we can quote only the name + yield f"grant {GRANT_BY_KIND[kind]} {to_identifier(schema)}.{object_name}" diff --git a/src/snowcli/cli/appify/util.py b/src/snowcli/cli/appify/util.py index fd6a182365..87ad7ece72 100644 --- a/src/snowcli/cli/appify/util.py +++ b/src/snowcli/cli/appify/util.py @@ -2,14 +2,14 @@ from click import ClickException from typing import Callable, Optional, List, Tuple from snowflake.connector.cursor import DictCursor -from snowcli.cli.project.util import DB_SCHEMA_AND_NAME +from snowcli.cli.project.util import DB_SCHEMA_AND_NAME, SCHEMA_AND_NAME STAGE_IMPORT_REGEX = f"@({DB_SCHEMA_AND_NAME})/" -class NotAFullyQualifiedNameError(ClickException): +class NotAQualifiedNameError(ClickException): def __init__(self, identifier: str): - super().__init__(f"Not a fully-qualified name: {identifier}") + super().__init__(f"Not an appropriately-qualified name: {identifier}") def find_row(cursor: DictCursor, predicate: Callable[[dict], bool]) -> Optional[dict]: @@ -37,4 +37,14 @@ def split_fqn_id(id: str) -> Tuple[str, str, str]: """ if match := re.fullmatch(DB_SCHEMA_AND_NAME, id): return (match.group(1), match.group(2), match.group(3)) - raise NotAFullyQualifiedNameError(id) + raise NotAQualifiedNameError(id) + + +def split_schema_and_object_id(id: str) -> Tuple[str, str, str]: + """ + Splits a partially-qualified identifier into its consituent parts. + Returns (schema, name); quoting carries over from the input. + """ + if match := re.fullmatch(DB_SCHEMA_AND_NAME, id): + return (match.group(1), match.group(2)) + raise NotAQualifiedNameError(id)