[sc-29679] RequestsConfig (#1031)

metaphor/common/docs/requests.md

@@ -0,0 +1,5 @@
+# Requests Config
+
+Controls configurable values for HTTP requests:
+
+- `timeout`: How many seconds before the HTTP client times out.

metaphor/common/requests_config.py

@@ -0,0 +1,15 @@
+from pydantic.dataclasses import dataclass
+
+from metaphor.common.dataclass import ConnectorConfig
+
+
+@dataclass(config=ConnectorConfig)
+class RequestsConfig:
+    """
+    Contains configuration values regarding HTTP requests.
+    """
+
+    timeout: int = 10
+    """
+    How many seconds before the requests client fails on a timed out request.
+    """

metaphor/fivetran/README.md

@@ -24,6 +24,10 @@ api_secret: <api_secret>
 
 See [Output Config](../common/docs/output.md) for more information.
 
+#### Requests configuration
+
+See [Requests Config](../common/docs/requests.md) for more information.
+
 ## Testing
 
 Follow the [Installation](../../README.md) instructions to install `metaphor-connectors` in your environment (or virtualenv).

metaphor/fivetran/config.py

@@ -5,6 +5,7 @@
 from metaphor.common.base_config import BaseConfig
 from metaphor.common.dataclass import ConnectorConfig
 from metaphor.common.filter import DatasetFilter
+from metaphor.common.requests_config import RequestsConfig
 
 
 @dataclass(config=ConnectorConfig)
@@ -14,3 +15,5 @@ class FivetranRunConfig(BaseConfig):
 
     # Include or exclude specific databases/schemas/tables
     filter: DatasetFilter = dataclass_field(default_factory=lambda: DatasetFilter())
+
+    requests: RequestsConfig = dataclass_field(default_factory=lambda: RequestsConfig())

metaphor/fivetran/extractor.py

@@ -135,6 +135,7 @@ def __init__(self, config: FivetranRunConfig) -> None:
         self._source_metadata: Dict[str, SourceMetadataPayload] = {}
         self._users: Dict[str, str] = {}
         self._base_url = "https://api.fivetran.com/v1"
+        self._requests_timeout = config.requests.timeout
 
     async def extract(self) -> Collection[ENTITY_TYPES]:
         logger.info("Fetching metadata from Fivetran")
@@ -549,4 +550,10 @@ def _get_all(self, url: str, type_: Type[DataT]) -> List[DataT]:
 
     def _call_get(self, url: str, **kwargs):
         headers = {"Accept": "application/json;version=2"}
-        return make_request(url=url, headers=headers, auth=self._auth, **kwargs)
+        return make_request(
+            url=url,
+            headers=headers,
+            timeout=self._requests_timeout,
+            auth=self._auth,
+            **kwargs,
+        )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.145"
+version = "0.14.146"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]

tests/common/test_api_request.py

@@ -1,6 +1,8 @@
 from typing import Dict
 from unittest.mock import MagicMock, patch
 
+import pytest
+import requests
 from pydantic import BaseModel
 
 from metaphor.common.api_request import ApiError, make_request
@@ -32,3 +34,19 @@ def test_get_request_not_200(mock_get: MagicMock):
         assert False, "ApiError not thrown"
     except ApiError:
         assert True
+
+
+def test_requests_timeout():
+    # Simple demo webserver to test delayed responses
+    # https://httpbin.org/#/Dynamic_data/get_delay__delay_
+    url = "https://httpbin.org/delay/2"
+
+    # This times out
+    with pytest.raises(requests.Timeout):
+        make_request(url, headers={"accept": "application/json"}, type_=Dict, timeout=1)
+
+    # This is OK
+    resp = make_request(
+        url, headers={"accept": "application/json"}, type_=Dict, timeout=3
+    )
+    assert resp