Merge branch 'main' into tsung-julii/sc-23517/hive-crawler-table-stat…

…s-column-stats
MetaphorData · Jan 9, 2024 · a20008b · a20008b
2 parents 0868972 + c354d18
commit a20008b
Show file tree

Hide file tree

Showing 6 changed files with 85 additions and 37 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -66,6 +66,10 @@ jobs:
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
 
+      - name: Setup dev libs
+        run: |
+          sudo apt-get install libsasl2-dev
+
       - name: Test
         run: |
           poetry install -E all

diff --git a/metaphor/monte_carlo/extractor.py b/metaphor/monte_carlo/extractor.py
@@ -43,7 +43,7 @@
     "ERROR": DataMonitorStatus.ERROR,
     "IN_PROGRESS": DataMonitorStatus.UNKNOWN,
     "NO_STATUS": DataMonitorStatus.UNKNOWN,
-    "MISCONFIGURED": DataMonitorStatus.ERROR,
+    "MISCONFIGURED": DataMonitorStatus.UNKNOWN,
     "IN_TRAINING": DataMonitorStatus.UNKNOWN,
 }
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.13.101"
+version = "0.13.103"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]

diff --git a/tests/monte_carlo/expected.json b/tests/monte_carlo/expected.json
@@ -7,7 +7,7 @@
           "lastRun": "2023-06-23T03:54:35.817000+00:00",
           "owner": "[email protected]",
           "severity": "UNKNOWN",
-          "status": "ERROR",
+          "status": "UNKNOWN",
           "targets": [],
           "title": "auto_monitor_name_cd5b69bd-e465-4545-b3f9-a5d507ea766c",
           "url": "https://getmontecarlo.com/monitors/e0dc143e-dd8a-4cb9-b4cc-dedec715d955"
@@ -53,5 +53,29 @@
       "name": "db.metaphor.test2",
       "platform": "SNOWFLAKE"
     }
+  },
+  {
+    "dataQuality": {
+      "monitors": [
+        {
+          "description": "Field Health for all fields in db:metaphor.test3",
+          "lastRun": "2023-06-23T03:54:35.817000+00:00",
+          "owner": "[email protected]",
+          "severity": "UNKNOWN",
+          "status": "ERROR",
+          "targets": [],
+          "title": "auto_monitor_name_18637195-a3c4-416e-a3e2-a89cc10adbc8",
+          "url": "https://getmontecarlo.com/monitors/2c156c8d-ab4a-432f-b8bb-f9ea9f31ed3d"
+        }
+      ],
+      "provider": "MONTE_CARLO",
+      "url": "https://getmontecarlo.com/assets/MCON++6418a1e2-9718-4413-9d2b-6a354e01ddf8++a19e22b4-7659-4064-8fd4-8d6122fabe1c++table++db:metaphor.test3/custom-monitors"
+    },
+    "id": "DATASET~EC9202BF4193DCE73F8EF04D5CAEAF75",
+    "logicalId": {
+      "account": "snow",
+      "name": "db.metaphor.test3",
+      "platform": "SNOWFLAKE"
+    }
   }
 ]
diff --git a/tests/monte_carlo/test_extractor.py b/tests/monte_carlo/test_extractor.py
@@ -37,6 +37,12 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                             "warehouse": {"connection_type": "SNOWFLAKE"},
                         }
                     },
+                    {
+                        "node": {
+                            "mcon": "MCON++6418a1e2-9718-4413-9d2b-6a354e01ddf8++a19e22b4-7659-4064-8fd4-8d6122fabe1c++table++db:metaphor.test3",
+                            "warehouse": {"connection_type": "SNOWFLAKE"},
+                        }
+                    },
                 ],
                 "page_info": {
                     "end_corsor": "cursor",
@@ -74,6 +80,20 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                     "creatorId": "[email protected]",
                     "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",
                 },
+                {
+                    "uuid": "2c156c8d-ab4a-432f-b8bb-f9ea9f31ed3d",
+                    "name": "auto_monitor_name_18637195-a3c4-416e-a3e2-a89cc10adbc8",
+                    "description": "Field Health for all fields in db:metaphor.test3",
+                    "entities": ["db:metaphor.test3"],
+                    "entityMcons": [
+                        "MCON++6418a1e2-9718-4413-9d2b-6a354e01ddf8++a19e22b4-7659-4064-8fd4-8d6122fabe1c++table++db:metaphor.test3"
+                    ],
+                    "severity": None,
+                    "monitorStatus": "ERROR",
+                    "monitorFields": None,
+                    "creatorId": "[email protected]",
+                    "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",
+                },
             ]
         },
     ]