From 6b9eb299e91612b1c16128f4c28768de89bc8b82 Mon Sep 17 00:00:00 2001 From: alyiwang Date: Thu, 18 Apr 2024 11:35:45 -0700 Subject: [PATCH 1/2] Not use SHOW SHARES [sc-25711] --- metaphor/snowflake/README.md | 5 +---- metaphor/snowflake/extractor.py | 5 +---- pyproject.toml | 4 ++-- tests/snowflake/test_extractor.py | 20 +------------------- 4 files changed, 5 insertions(+), 29 deletions(-) diff --git a/metaphor/snowflake/README.md b/metaphor/snowflake/README.md index 36279e2e..d31af19d 100644 --- a/metaphor/snowflake/README.md +++ b/metaphor/snowflake/README.md @@ -28,12 +28,9 @@ grant usage on warehouse identifier($warehouse) to role identifier($role); -- Grant privilege to access Snowflake Account Usage views: grant imported privileges on database snowflake to role identifier($role); - --- (Optional) Grant privilege to "show shares" for inbound shared databases -grant import share on account to identifier($role); ``` -For each database, run the following statements to grant the required privileges: +For each database, including the inbound shared databases, run the following statements to grant the required privileges: ```sql set db = ''; diff --git a/metaphor/snowflake/extractor.py b/metaphor/snowflake/extractor.py index adaab539..bf10a8f6 100644 --- a/metaphor/snowflake/extractor.py +++ b/metaphor/snowflake/extractor.py @@ -186,13 +186,10 @@ def fetch_databases(cursor: SnowflakeCursor) -> List[str]: @staticmethod def _fetch_shared_databases(cursor: SnowflakeCursor) -> List[str]: - cursor.execute("SHOW SHARES") - shared_database = [db[4].lower() for db in cursor if db[1] == "INBOUND"] cursor.execute( "SELECT database_name FROM information_schema.databases WHERE type = 'IMPORTED DATABASE' ORDER BY database_name" ) - shared_database += [db[0].lower() for db in cursor] - return list(set(shared_database)) + return [db[0].lower() for db in cursor] @staticmethod def _fetch_secure_views(cursor: SnowflakeCursor) -> Set[str]: diff --git a/pyproject.toml b/pyproject.toml index 0779f1f0..09bfbf72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.13.168" +version = "0.13.169" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "] @@ -200,4 +200,4 @@ skips = [ 'B603', 'B607', 'B608' -] \ No newline at end of file +] diff --git a/tests/snowflake/test_extractor.py b/tests/snowflake/test_extractor.py index 67793889..ee5e80dd 100644 --- a/tests/snowflake/test_extractor.py +++ b/tests/snowflake/test_extractor.py @@ -476,31 +476,13 @@ def test_fetch_shared_databases(mock_connect: MagicMock): mock_cursor = MagicMock() mock_cursor.__iter__.side_effect = [ - iter( - [ - ( - None, - "INBOUND", - None, - None, - "shared_1", - ), - ( - None, - "UNKNOWN", - None, - None, - "shared_2", - ), - ] - ), iter([("shared_1",), ("shared_3",)]), ] extractor = SnowflakeExtractor(make_snowflake_config()) results = extractor._fetch_shared_databases(mock_cursor) - assert set(results) == {"shared_1", "shared_3"} + assert results == ["shared_1", "shared_3"] @patch("metaphor.snowflake.extractor.check_access_history") From f7379e1bf2d161ae4ffc7557b69c25d36b9bcee9 Mon Sep 17 00:00:00 2001 From: alyiwang Date: Thu, 18 Apr 2024 14:18:16 -0700 Subject: [PATCH 2/2] fetch all databases --- metaphor/snowflake/extractor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metaphor/snowflake/extractor.py b/metaphor/snowflake/extractor.py index bf10a8f6..4f6f9e56 100644 --- a/metaphor/snowflake/extractor.py +++ b/metaphor/snowflake/extractor.py @@ -179,8 +179,11 @@ def collect_query_logs(self) -> Iterator[QueryLog]: @staticmethod def fetch_databases(cursor: SnowflakeCursor) -> List[str]: + """ + Fetch all databases from Snowflake, including shared/imported databases. + """ cursor.execute( - "SELECT database_name FROM information_schema.databases WHERE type != 'IMPORTED DATABASE' ORDER BY database_name" + "SELECT database_name FROM information_schema.databases ORDER BY database_name" ) return [db[0].lower() for db in cursor]