From b43330db0a907180ee5d7b751bec1702275df4bf Mon Sep 17 00:00:00 2001
From: Mars Lan <mars@metaphor.io>
Date: Fri, 20 Dec 2024 13:52:18 -0800
Subject: [PATCH] Add pagination to Monte Carlo getMonitors call

---
 metaphor/monte_carlo/extractor.py | 33 +++++++++++++++++++------------
 pyproject.toml                    |  2 +-
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/metaphor/monte_carlo/extractor.py b/metaphor/monte_carlo/extractor.py
index 7c1ee0b0..2eea281c 100644
--- a/metaphor/monte_carlo/extractor.py
+++ b/metaphor/monte_carlo/extractor.py
@@ -1,5 +1,4 @@
 import re
-import traceback
 from typing import Collection, Dict, List
 
 from dateutil import parser
@@ -102,11 +101,15 @@ def _fetch_monitors(self) -> None:
         See https://apidocs.getmontecarlo.com/#query-getMonitors
         """
 
-        try:
-            monitors = self._client(
+        offset = 0
+        limit = 200
+
+        monitors = []
+        while True:
+            resp = self._client(
                 """
-                {
-                  getMonitors {
+                query getMonitors($offset: Int, $limit: Int) {
+                  getMonitors(offset: $offset, limit: $limit) {
                     uuid
                     name
                     description
@@ -120,16 +123,20 @@ def _fetch_monitors(self) -> None:
                     exceptions
                   }
                 }
-                """
+                """,
+                {"offset": offset, "limit": limit},
             )
 
-            logger.info(f"Fetched {len(monitors['get_monitors'])} monitors")
-            json_dump_to_debug_file(monitors, "getMonitors.json")
+            logger.info(f"Querying getMonitors with offset {offset}")
+            monitors.extend(resp["get_monitors"])
+            if len(resp["get_monitors"]) < limit:
+                break
+
+            offset += limit
 
-            self._parse_monitors(monitors)
-        except Exception as error:
-            traceback.print_exc()
-            logger.error(f"Failed to get all monitors, error {error}")
+        logger.info(f"Fetched {len(monitors)} monitors")
+        json_dump_to_debug_file(monitors, "getMonitors.json")
+        self._parse_monitors(monitors)
 
     def _fetch_tables(self) -> None:
         """Fetch all tables
@@ -188,7 +195,7 @@ def _fetch_tables(self) -> None:
                 self._mcon_platform_map[mcon] = platform
 
     def _parse_monitors(self, monitors) -> None:
-        for monitor in monitors["get_monitors"]:
+        for monitor in monitors:
             uuid = monitor["uuid"]
             monitor_severity = monitor_severity_map.get(
                 monitor["priority"], DataMonitorSeverity.UNKNOWN
diff --git a/pyproject.toml b/pyproject.toml
index 4ef3d955..f6561cfd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.161"
+version = "0.14.162"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <dev@metaphor.io>"]