reorganize page_inputs.py as a submodule; move HttpClient to it

tests/test_requests.py

@@ -3,15 +3,13 @@
 import pytest
 from web_poet.exceptions import RequestBackendError
 from web_poet.page_inputs import (
+    HttpClient,
     HttpRequest,
     HttpResponse,
     HttpRequestBody,
     HttpRequestHeaders
 )
-from web_poet.requests import (
-    HttpClient,
-    request_backend_var,
-)
+from web_poet.requests import request_backend_var
 
 
 @pytest.fixture
@@ -47,7 +45,7 @@ async def test_perform_request_from_httpclient(async_mock):
 async def test_http_client_single_requests(async_mock):
     client = HttpClient(async_mock)
 
-    with mock.patch("web_poet.requests.HttpRequest") as mock_request:
+    with mock.patch("web_poet.page_inputs.client.HttpRequest") as mock_request:
         response = await client.request("url")
         response.url == "url"
 

web_poet/__init__.py

@@ -1,10 +1,8 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
-from .requests import (
-    request_backend_var,
-    HttpClient,
-)
+from .requests import request_backend_var
 from .page_inputs import (
     Meta,
+    HttpClient,
     HttpRequest,
     HttpResponse,
     HttpRequestHeaders,

web_poet/page_inputs/__init__.py

@@ -0,0 +1,10 @@
+from .meta import Meta
+from .client import HttpClient
+from .http import (
+    HttpRequest,
+    HttpResponse,
+    HttpRequestHeaders,
+    HttpResponseHeaders,
+    HttpRequestBody,
+    HttpResponseBody,
+)

web_poet/page_inputs/client.py

@@ -0,0 +1,123 @@
+"""This module has a full support for :mod:`asyncio` that enables developers to
+perform asynchronous additional requests inside of Page Objects.
+
+Note that the implementation to fully execute any :class:`~.Request` is not
+handled in this module. With that, the framework using **web-poet** must supply
+the implementation.
+
+You can read more about this in the :ref:`advanced-downloader-impl` documentation.
+"""
+
+import asyncio
+from typing import Optional, Dict, List, Union, Callable
+
+from web_poet.requests import request_backend_var, _perform_request
+from web_poet.page_inputs.http import (
+    HttpRequest,
+    HttpRequestHeaders,
+    HttpRequestBody,
+    HttpResponse,
+)
+
+
+_StrMapping = Dict[str, str]
+_Headers = Union[_StrMapping, HttpRequestHeaders]
+_Body = Union[bytes, HttpRequestBody]
+
+
+class HttpClient:
+    """A convenient client to easily execute requests.
+
+    By default, it uses the request implementation assigned in the
+    ``web_poet.request_backend_var`` which is a :mod:`contextvars` instance to
+    download the actual requests. However, it can easily be overridable by
+    providing an optional ``request_downloader`` callable.
+
+    Providing the request implementation by dependency injection would be a good
+    alternative solution when you want to avoid setting up :mod:`contextvars`
+    like ``web_poet.request_backend_var``.
+
+    In any case, this doesn't contain any implementation about how to execute
+    any requests fed into it. When setting that up, make sure that the downloader
+    implementation returns a :class:`~.HttpResponse` instance.
+    """
+
+    def __init__(self, request_downloader: Callable = None):
+        self._request_downloader = request_downloader or _perform_request
+
+    async def request(
+        self,
+        url: str,
+        *,
+        method: str = "GET",
+        headers: Optional[_Headers] = None,
+        body: Optional[_Body] = None,
+    ) -> HttpResponse:
+        """This is a shortcut for creating a :class:`~.HttpRequest` instance and executing
+        that request.
+
+        A :class:`~.HttpResponse` instance should then be returned.
+
+        .. warning::
+            By convention, the request implementation supplied optionally to
+            :class:`~.HttpClient` should return a :class:`~.HttpResponse` instance.
+            However, the underlying implementation supplied might change that,
+            depending on how the framework using **web-poet** implements it.
+        """
+        headers = headers or {}
+        body = body or b""
+        req = HttpRequest(url=url, method=method, headers=headers, body=body)
+        return await self.execute(req)
+
+    async def get(
+        self, url: str, *, headers: Optional[_Headers] = None
+    ) -> HttpResponse:
+        """Similar to :meth:`~.HttpClient.request` but peforming a ``GET``
+        request.
+        """
+        return await self.request(url=url, method="GET", headers=headers)
+
+    async def post(
+        self,
+        url: str,
+        *,
+        headers: Optional[_Headers] = None,
+        body: Optional[_Body] = None,
+    ) -> HttpResponse:
+        """Similar to :meth:`~.HttpClient.request` but performing a ``POST``
+        request.
+        """
+        return await self.request(url=url, method="POST", headers=headers, body=body)
+
+    async def execute(self, request: HttpRequest) -> HttpResponse:
+        """Accepts a single instance of :class:`~.HttpRequest` and executes it
+        using the request implementation configured in the :class:`~.HttpClient`
+        instance.
+
+        This returns a single :class:`~.HttpResponse`.
+        """
+        return await self._request_downloader(request)
+
+    async def batch_execute(
+        self, *requests: HttpRequest, return_exceptions: bool = False
+    ) -> List[Union[HttpResponse, Exception]]:
+        """Similar to :meth:`~.HttpClient.execute` but accepts a collection of
+        :class:`~.HttpRequest` instances that would be batch executed.
+
+        The order of the :class:`~.HttpResponses` would correspond to the order
+        of :class:`~.HttpRequest` passed.
+
+        If any of the :class:`~.HttpRequest` raises an exception upon execution,
+        the exception is raised.
+
+        To prevent this, the actual exception can be returned alongside any
+        successful :class:`~.HttpResponse`. This enables salvaging any usable
+        responses despite any possible failures. This can be done by setting
+        ``True`` to the ``return_exceptions`` parameter.
+        """
+
+        coroutines = [self._request_downloader(r) for r in requests]
+        responses = await asyncio.gather(
+            *coroutines, return_exceptions=return_exceptions
+        )
+        return responses

web_poet/page_inputs.py → web_poet/page_inputs/http.py

@@ -14,7 +14,8 @@
 from web_poet.utils import memoizemethod_noargs
 
 T_headers = TypeVar("T_headers", bound="HttpResponseHeaders")
-AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
+
+_AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
 
 
 class HttpRequestBody(bytes):
@@ -99,7 +100,7 @@ class HttpResponseHeaders(_HttpHeaders):
 
     @classmethod
     def from_bytes_dict(
-        cls: Type[T_headers], arg: AnyStrDict, encoding: str = "utf-8"
+        cls: Type[T_headers], arg: _AnyStrDict, encoding: str = "utf-8"
     ) -> T_headers:
         """An alternative constructor for instantiation where the header-value
         pairs could be in raw bytes form.
@@ -270,13 +271,3 @@ def _auto_detect_fun(self, body: bytes) -> Optional[str]:
             except UnicodeError:
                 continue
             return resolve_encoding(enc)
-
-
-class Meta(dict):
-    """Container class that could contain any arbitrary data to be passed into
-    a Page Object.
-
-    Note that this is simply a subclass of Python's ``dict``.
-    """
-
-    pass

web_poet/page_inputs/meta.py

@@ -0,0 +1,8 @@
+class Meta(dict):
+    """Container class that could contain any arbitrary data to be passed into
+    a Page Object.
+
+    Note that this is simply a subclass of Python's ``dict``.
+    """
+
+    pass

web_poet/requests.py

@@ -1,35 +1,14 @@
-"""This module has a full support for :mod:`asyncio` that enables developers to
-perform asynchronous additional requests inside of Page Objects.
-
-Note that the implementation to fully execute any :class:`~.Request` is not
-handled in this module. With that, the framework using **web-poet** must supply
-the implementation.
-
-You can read more about this in the :ref:`advanced-downloader-impl` documentation.
-"""
-
-import asyncio
 import logging
 from contextvars import ContextVar
-from typing import Optional, List, Callable, Union, Dict
-
-import attrs
 
-from web_poet.page_inputs import (
-    HttpResponse,
+from web_poet.exceptions import RequestBackendError
+from web_poet.page_inputs.http import (
     HttpRequest,
-    HttpRequestHeaders,
-    HttpRequestBody,
+    HttpResponse,
 )
-from web_poet.exceptions import RequestBackendError
 
 logger = logging.getLogger(__name__)
 
-_StrMapping = Dict[str, str]
-_Headers = Union[_StrMapping, HttpRequestHeaders]
-_Body = Union[bytes, HttpRequestBody]
-
-
 # Frameworks that wants to support additional requests in ``web-poet`` should
 # set the appropriate implementation for requesting data.
 request_backend_var: ContextVar = ContextVar("request_backend")
@@ -60,99 +39,3 @@ async def _perform_request(request: HttpRequest) -> HttpResponse:
 
     response_data: HttpResponse = await request_backend(request)
     return response_data
-
-
-class HttpClient:
-    """A convenient client to easily execute requests.
-
-    By default, it uses the request implementation assigned in the
-    ``web_poet.request_backend_var`` which is a :mod:`contextvars` instance to
-    download the actual requests. However, it can easily be overridable by
-    providing an optional ``request_downloader`` callable.
-
-    Providing the request implementation by dependency injection would be a good
-    alternative solution when you want to avoid setting up :mod:`contextvars`
-    like ``web_poet.request_backend_var``.
-
-    In any case, this doesn't contain any implementation about how to execute
-    any requests fed into it. When setting that up, make sure that the downloader
-    implementation returns a :class:`~.HttpResponse` instance.
-    """
-
-    def __init__(self, request_downloader: Callable = None):
-        self._request_downloader = request_downloader or _perform_request
-
-    async def request(
-        self,
-        url: str,
-        *,
-        method: str = "GET",
-        headers: Optional[_Headers] = None,
-        body: Optional[_Body] = None,
-    ) -> HttpResponse:
-        """This is a shortcut for creating a :class:`~.HttpRequest` instance and executing
-        that request.
-
-        A :class:`~.HttpResponse` instance should then be returned.
-
-        .. warning::
-            By convention, the request implementation supplied optionally to
-            :class:`~.HttpClient` should return a :class:`~.HttpResponse` instance.
-            However, the underlying implementation supplied might change that,
-            depending on how the framework using **web-poet** implements it.
-        """
-        headers = headers or {}
-        body = body or b""
-        req = HttpRequest(url=url, method=method, headers=headers, body=body)
-        return await self.execute(req)
-
-    async def get(self, url: str, *, headers: Optional[_Headers] = None) -> HttpResponse:
-        """Similar to :meth:`~.HttpClient.request` but peforming a ``GET``
-        request.
-        """
-        return await self.request(url=url, method="GET", headers=headers)
-
-    async def post(
-        self,
-        url: str,
-        *,
-        headers: Optional[_Headers] = None,
-        body: Optional[_Body] = None,
-    ) -> HttpResponse:
-        """Similar to :meth:`~.HttpClient.request` but performing a ``POST``
-        request.
-        """
-        return await self.request(url=url, method="POST", headers=headers, body=body)
-
-    async def execute(self, request: HttpRequest) -> HttpResponse:
-        """Accepts a single instance of :class:`~.HttpRequest` and executes it
-        using the request implementation configured in the :class:`~.HttpClient`
-        instance.
-
-        This returns a single :class:`~.HttpResponse`.
-        """
-        return await self._request_downloader(request)
-
-    async def batch_execute(
-        self, *requests: HttpRequest, return_exceptions: bool = False
-    ) -> List[Union[HttpResponse, Exception]]:
-        """Similar to :meth:`~.HttpClient.execute` but accepts a collection of
-        :class:`~.HttpRequest` instances that would be batch executed.
-
-        The order of the :class:`~.HttpResponses` would correspond to the order
-        of :class:`~.HttpRequest` passed.
-
-        If any of the :class:`~.HttpRequest` raises an exception upon execution,
-        the exception is raised.
-
-        To prevent this, the actual exception can be returned alongside any
-        successful :class:`~.HttpResponse`. This enables salvaging any usable
-        responses despite any possible failures. This can be done by setting
-        ``True`` to the ``return_exceptions`` parameter.
-        """
-
-        coroutines = [self._request_downloader(r) for r in requests]
-        responses = await asyncio.gather(
-            *coroutines, return_exceptions=return_exceptions
-        )
-        return responses