From a53c91157edc86f521d9dea81f943f407875eb43 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Wed, 18 Dec 2024 02:55:04 +1100 Subject: [PATCH 1/2] docs: add hint about connection pooling --- .../docs/general-usage/http/rest-client.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/website/docs/general-usage/http/rest-client.md b/docs/website/docs/general-usage/http/rest-client.md index 53e1a2b7c5..5ae5d131c4 100644 --- a/docs/website/docs/general-usage/http/rest-client.md +++ b/docs/website/docs/general-usage/http/rest-client.md @@ -693,3 +693,30 @@ for page in client.paginate( ): print(page) ``` + +## Performance + +If you have a substantial amount of requests going to a single or smaller set of hosts, it can be worth to use connection pooling. +You can do this by leveraging the custom `session` parameter like so: + +```py +from dlt.sources.helpers.rest_client.client import RESTClient +from dlt.sources.helpers.requests.session import Session +from requests.adapters import HTTPAdapter + +base_url = "https://api.my-service.com" + +session = Session(raise_for_status=False) +adapter = HTTPAdapter( + # We're only using one host for all requests in our example + pool_connections=1, + # But we're making a lot of requests and want to reuse connections + pool_maxsize=100, +) +session.mount(api_base, adapter) + +client = RESTClient( + base_url=base_url, + session=session, +) +``` From a18cc335a32235579984f326f0290ff0e09ca1b8 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Wed, 18 Dec 2024 02:55:59 +1100 Subject: [PATCH 2/2] Update docs/website/docs/general-usage/http/rest-client.md --- docs/website/docs/general-usage/http/rest-client.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website/docs/general-usage/http/rest-client.md b/docs/website/docs/general-usage/http/rest-client.md index 5ae5d131c4..101bffd17e 100644 --- a/docs/website/docs/general-usage/http/rest-client.md +++ b/docs/website/docs/general-usage/http/rest-client.md @@ -713,7 +713,7 @@ adapter = HTTPAdapter( # But we're making a lot of requests and want to reuse connections pool_maxsize=100, ) -session.mount(api_base, adapter) +session.mount(base_url, adapter) client = RESTClient( base_url=base_url,