[sc-25443] Store data quality monitor failures (#983)

* [sc-25443] Store data quality monitor failures * add poetry.lock
MetaphorData · Sep 13, 2024 · e011efc · e011efc
1 parent f92e2d1
commit e011efc
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 14 deletions.
diff --git a/metaphor/monte_carlo/extractor.py b/metaphor/monte_carlo/extractor.py
@@ -44,7 +44,7 @@
     "ERROR": DataMonitorStatus.ERROR,
     "IN_PROGRESS": DataMonitorStatus.UNKNOWN,
     "NO_STATUS": DataMonitorStatus.UNKNOWN,
-    "MISCONFIGURED": DataMonitorStatus.UNKNOWN,
+    "MISCONFIGURED": DataMonitorStatus.ERROR,
     "IN_TRAINING": DataMonitorStatus.UNKNOWN,
 }
 
@@ -117,6 +117,7 @@ def _fetch_monitors(self) -> None:
                     monitorFields
                     creatorId
                     prevExecutionTime
+                    exceptions
                   }
                 }
                 """
@@ -193,6 +194,11 @@ def _parse_monitors(self, monitors) -> None:
                 monitor["priority"], DataMonitorSeverity.UNKNOWN
             )
 
+            # MC monitor exceptions is a single string
+            exceptions = (
+                [monitor["exceptions"]] if monitor["exceptions"] is not None else None
+            )
+
             data_monitor = DataMonitor(
                 title=monitor["name"],
                 description=monitor["description"],
@@ -205,6 +211,7 @@ def _parse_monitors(self, monitors) -> None:
                     DataMonitorTarget(column=field.upper())
                     for field in monitor["monitorFields"] or []
                 ],
+                exceptions=exceptions,
             )
 
             if monitor["entities"] is None or monitor["entityMcons"] is None:
@@ -233,7 +240,9 @@ def _parse_monitor_status(self, monitor: Dict):
         )
 
         # Change status to UNKNOWN if the error message is to be ignored
-        message = monitor.get("exceptions", "")
+        message = (
+            monitor.get("exceptions", "") or ""
+        )  # monitor["exceptions"] is default to None
         if status == DataMonitorStatus.ERROR:
             for ignored_error in self._ignored_errors:
                 if re.match(ignored_error, message) is not None:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.104"
+version = "0.14.105"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]
@@ -42,7 +42,7 @@ llama-index-readers-confluence = { version = "^0.1.4", optional = true }
 llama-index-readers-notion = { version = "^0.1.6", optional = true }
 looker-sdk = { version = "^24.2.0", optional = true }
 lxml = { version = "~=5.0.0", optional = true }
-metaphor-models = "0.38.1"
+metaphor-models = "0.38.2"
 more-itertools = { version = "^10.1.0", optional = true }
 msal = { version = "^1.28.0", optional = true }
 msgraph-beta-sdk = { version = "~1.4.0", optional = true }

diff --git a/tests/monte_carlo/expected.json b/tests/monte_carlo/expected.json
@@ -7,7 +7,7 @@
           "lastRun": "2023-06-23T03:54:35.817000+00:00",
           "owner": "[email protected]",
           "severity": "HIGH",
-          "status": "UNKNOWN",
+          "status": "ERROR",
           "targets": [],
           "title": "auto_monitor_name_cd5b69bd-e465-4545-b3f9-a5d507ea766c",
           "url": "https://getmontecarlo.com/monitors/e0dc143e-dd8a-4cb9-b4cc-dedec715d955"
@@ -83,10 +83,13 @@
       "monitors": [
         {
           "description": "Field Health for all fields in db:metaphor.test4",
+          "exceptions": [
+            "Oh no"
+          ],
           "lastRun": "2023-06-23T03:54:35.817000+00:00",
           "owner": "[email protected]",
           "severity": "UNKNOWN",
-          "status": "UNKNOWN",
+          "status": "ERROR",
           "targets": [],
           "title": "auto_monitor_name_693b98e3-950d-472b-83fe-8c8e5b5979f9",
           "url": "https://getmontecarlo.com/monitors/d14af7d8-6342-420a-bb09-5805fad677f1"

diff --git a/tests/monte_carlo/test_extractor.py b/tests/monte_carlo/test_extractor.py
@@ -71,6 +71,7 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                     "monitorFields": None,
                     "creatorId": "[email protected]",
                     "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",
+                    "exceptions": None,
                 },
                 {
                     "uuid": "ce4c4568-35f4-4365-a6fe-95f233fcf6c3",
@@ -85,6 +86,7 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                     "monitorFields": ["foo", "bar"],
                     "creatorId": "[email protected]",
                     "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",
+                    "exceptions": None,
                 },
                 {
                     "uuid": "2c156c8d-ab4a-432f-b8bb-f9ea9f31ed3d",
@@ -99,6 +101,7 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                     "monitorFields": None,
                     "creatorId": "[email protected]",
                     "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",
+                    "exceptions": None,
                 },
                 {
                     "uuid": "d14af7d8-6342-420a-bb09-5805fad677f1",
@@ -110,7 +113,7 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                     ],
                     "priority": None,
                     "monitorStatus": "ERROR",
-                    "exceptions": "Ignore me",
+                    "exceptions": "Oh no",
                     "monitorFields": None,
                     "creatorId": "[email protected]",
                     "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",
@@ -123,7 +126,7 @@ async def test_extractor(mock_pycarlo_client: MagicMock, test_root_dir: str):
                     "entityMcons": None,
                     "priority": None,
                     "monitorStatus": "ERROR",
-                    "exceptions": "Ignore me",
+                    "exceptions": "No entity, this exception is ignored",
                     "monitorFields": None,
                     "creatorId": "[email protected]",
                     "prevExecutionTime": "2023-06-23T03:54:35.817000+00:00",