Add paginate to dlt.sources.helpers.requests

dlt/sources/helpers/requests/__init__.py

@@ -15,6 +15,7 @@
 from requests.exceptions import ChunkedEncodingError
 from dlt.sources.helpers.requests.retry import Client
 from dlt.sources.helpers.requests.session import Session
+from dlt.sources.helpers.rest_client import paginate # noqa: F401
 from dlt.common.configuration.specs import RunConfiguration
 
 client = Client()

dlt/sources/helpers/rest_client/__init__.py

@@ -1,4 +1,4 @@
-from typing import Optional, Dict, Iterator, Union, Any
+from typing import Optional, Dict, Iterator, Any
 
 from dlt.common import jsonpath
 

dlt/sources/helpers/rest_client/auth.py

@@ -12,7 +12,7 @@
     Iterable,
     TYPE_CHECKING,
 )
-from dlt.sources.helpers import requests
+
 from requests.auth import AuthBase
 from requests import PreparedRequest  # noqa: I251
 import pendulum
@@ -160,6 +160,8 @@ def is_token_expired(self) -> bool:
         return not self.token_expiry or pendulum.now() >= self.token_expiry
 
     def obtain_token(self) -> None:
+        from dlt.sources.helpers import requests
+
         try:
             import jwt
         except ModuleNotFoundError:

dlt/sources/helpers/rest_client/client.py

@@ -16,7 +16,6 @@
 
 from dlt.common import logger
 from dlt.common import jsonpath
-from dlt.sources.helpers.requests.retry import Client
 
 from .typing import HTTPMethodBasic, HTTPMethod, Hooks
 from .paginators import BasePaginator
@@ -84,6 +83,7 @@ def __init__(
             self._validate_session_raise_for_status(session)
             self.session = session
         else:
+            from dlt.sources.helpers.requests.retry import Client
             self.session = Client(raise_for_status=False).session
 
         self.paginator = paginator