diff --git a/docs/website/docs/general-usage/http/rest-client.md b/docs/website/docs/general-usage/http/rest-client.md index 53e1a2b7c5..101bffd17e 100644 --- a/docs/website/docs/general-usage/http/rest-client.md +++ b/docs/website/docs/general-usage/http/rest-client.md @@ -693,3 +693,30 @@ for page in client.paginate( ): print(page) ``` + +## Performance + +If you have a substantial amount of requests going to a single or smaller set of hosts, it can be worth to use connection pooling. +You can do this by leveraging the custom `session` parameter like so: + +```py +from dlt.sources.helpers.rest_client.client import RESTClient +from dlt.sources.helpers.requests.session import Session +from requests.adapters import HTTPAdapter + +base_url = "https://api.my-service.com" + +session = Session(raise_for_status=False) +adapter = HTTPAdapter( + # We're only using one host for all requests in our example + pool_connections=1, + # But we're making a lot of requests and want to reuse connections + pool_maxsize=100, +) +session.mount(base_url, adapter) + +client = RESTClient( + base_url=base_url, + session=session, +) +```