From 652828945d2637d90769bab0b17e60f3caf8f5fd Mon Sep 17 00:00:00 2001
From: Tsung-Ju Lii <andy@metaphor.io>
Date: Mon, 4 Nov 2024 22:48:37 +0800
Subject: [PATCH] [sc-29679] `RequestsConfig` (#1031)

---
 metaphor/common/docs/requests.md   |  5 +++++
 metaphor/common/requests_config.py | 15 +++++++++++++++
 metaphor/fivetran/README.md        |  4 ++++
 metaphor/fivetran/config.py        |  3 +++
 metaphor/fivetran/extractor.py     |  9 ++++++++-
 pyproject.toml                     |  2 +-
 tests/common/test_api_request.py   | 18 ++++++++++++++++++
 7 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 metaphor/common/docs/requests.md
 create mode 100644 metaphor/common/requests_config.py

diff --git a/metaphor/common/docs/requests.md b/metaphor/common/docs/requests.md
new file mode 100644
index 00000000..cd7f726a
--- /dev/null
+++ b/metaphor/common/docs/requests.md
@@ -0,0 +1,5 @@
+# Requests Config
+
+Controls configurable values for HTTP requests:
+
+- `timeout`: How many seconds before the HTTP client times out.
diff --git a/metaphor/common/requests_config.py b/metaphor/common/requests_config.py
new file mode 100644
index 00000000..42dc26de
--- /dev/null
+++ b/metaphor/common/requests_config.py
@@ -0,0 +1,15 @@
+from pydantic.dataclasses import dataclass
+
+from metaphor.common.dataclass import ConnectorConfig
+
+
+@dataclass(config=ConnectorConfig)
+class RequestsConfig:
+    """
+    Contains configuration values regarding HTTP requests.
+    """
+
+    timeout: int = 10
+    """
+    How many seconds before the requests client fails on a timed out request.
+    """
diff --git a/metaphor/fivetran/README.md b/metaphor/fivetran/README.md
index 777347a8..cf38914f 100644
--- a/metaphor/fivetran/README.md
+++ b/metaphor/fivetran/README.md
@@ -24,6 +24,10 @@ api_secret: <api_secret>
 
 See [Output Config](../common/docs/output.md) for more information.
 
+#### Requests configuration
+
+See [Requests Config](../common/docs/requests.md) for more information.
+
 ## Testing
 
 Follow the [Installation](../../README.md) instructions to install `metaphor-connectors` in your environment (or virtualenv).
diff --git a/metaphor/fivetran/config.py b/metaphor/fivetran/config.py
index 067bb3b9..6b5d6a3c 100644
--- a/metaphor/fivetran/config.py
+++ b/metaphor/fivetran/config.py
@@ -5,6 +5,7 @@
 from metaphor.common.base_config import BaseConfig
 from metaphor.common.dataclass import ConnectorConfig
 from metaphor.common.filter import DatasetFilter
+from metaphor.common.requests_config import RequestsConfig
 
 
 @dataclass(config=ConnectorConfig)
@@ -14,3 +15,5 @@ class FivetranRunConfig(BaseConfig):
 
     # Include or exclude specific databases/schemas/tables
     filter: DatasetFilter = dataclass_field(default_factory=lambda: DatasetFilter())
+
+    requests: RequestsConfig = dataclass_field(default_factory=lambda: RequestsConfig())
diff --git a/metaphor/fivetran/extractor.py b/metaphor/fivetran/extractor.py
index 72a0a93b..8353167a 100644
--- a/metaphor/fivetran/extractor.py
+++ b/metaphor/fivetran/extractor.py
@@ -135,6 +135,7 @@ def __init__(self, config: FivetranRunConfig) -> None:
         self._source_metadata: Dict[str, SourceMetadataPayload] = {}
         self._users: Dict[str, str] = {}
         self._base_url = "https://api.fivetran.com/v1"
+        self._requests_timeout = config.requests.timeout
 
     async def extract(self) -> Collection[ENTITY_TYPES]:
         logger.info("Fetching metadata from Fivetran")
@@ -549,4 +550,10 @@ def _get_all(self, url: str, type_: Type[DataT]) -> List[DataT]:
 
     def _call_get(self, url: str, **kwargs):
         headers = {"Accept": "application/json;version=2"}
-        return make_request(url=url, headers=headers, auth=self._auth, **kwargs)
+        return make_request(
+            url=url,
+            headers=headers,
+            timeout=self._requests_timeout,
+            auth=self._auth,
+            **kwargs,
+        )
diff --git a/pyproject.toml b/pyproject.toml
index 09c0f9f2..246197c0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.145"
+version = "0.14.146"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <dev@metaphor.io>"]
diff --git a/tests/common/test_api_request.py b/tests/common/test_api_request.py
index f634cba1..0fa36d25 100644
--- a/tests/common/test_api_request.py
+++ b/tests/common/test_api_request.py
@@ -1,6 +1,8 @@
 from typing import Dict
 from unittest.mock import MagicMock, patch
 
+import pytest
+import requests
 from pydantic import BaseModel
 
 from metaphor.common.api_request import ApiError, make_request
@@ -32,3 +34,19 @@ def test_get_request_not_200(mock_get: MagicMock):
         assert False, "ApiError not thrown"
     except ApiError:
         assert True
+
+
+def test_requests_timeout():
+    # Simple demo webserver to test delayed responses
+    # https://httpbin.org/#/Dynamic_data/get_delay__delay_
+    url = "https://httpbin.org/delay/2"
+
+    # This times out
+    with pytest.raises(requests.Timeout):
+        make_request(url, headers={"accept": "application/json"}, type_=Dict, timeout=1)
+
+    # This is OK
+    resp = make_request(
+        url, headers={"accept": "application/json"}, type_=Dict, timeout=3
+    )
+    assert resp