diff --git a/metaphor/monte_carlo/extractor.py b/metaphor/monte_carlo/extractor.py index 7c1ee0b0..2eea281c 100644 --- a/metaphor/monte_carlo/extractor.py +++ b/metaphor/monte_carlo/extractor.py @@ -1,5 +1,4 @@ import re -import traceback from typing import Collection, Dict, List from dateutil import parser @@ -102,11 +101,15 @@ def _fetch_monitors(self) -> None: See https://apidocs.getmontecarlo.com/#query-getMonitors """ - try: - monitors = self._client( + offset = 0 + limit = 200 + + monitors = [] + while True: + resp = self._client( """ - { - getMonitors { + query getMonitors($offset: Int, $limit: Int) { + getMonitors(offset: $offset, limit: $limit) { uuid name description @@ -120,16 +123,20 @@ def _fetch_monitors(self) -> None: exceptions } } - """ + """, + {"offset": offset, "limit": limit}, ) - logger.info(f"Fetched {len(monitors['get_monitors'])} monitors") - json_dump_to_debug_file(monitors, "getMonitors.json") + logger.info(f"Querying getMonitors with offset {offset}") + monitors.extend(resp["get_monitors"]) + if len(resp["get_monitors"]) < limit: + break + + offset += limit - self._parse_monitors(monitors) - except Exception as error: - traceback.print_exc() - logger.error(f"Failed to get all monitors, error {error}") + logger.info(f"Fetched {len(monitors)} monitors") + json_dump_to_debug_file(monitors, "getMonitors.json") + self._parse_monitors(monitors) def _fetch_tables(self) -> None: """Fetch all tables @@ -188,7 +195,7 @@ def _fetch_tables(self) -> None: self._mcon_platform_map[mcon] = platform def _parse_monitors(self, monitors) -> None: - for monitor in monitors["get_monitors"]: + for monitor in monitors: uuid = monitor["uuid"] monitor_severity = monitor_severity_map.get( monitor["priority"], DataMonitorSeverity.UNKNOWN diff --git a/pyproject.toml b/pyproject.toml index 4ef3d955..f6561cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.14.161" +version = "0.14.162" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "]