diff --git a/tests/test_requests.py b/tests/test_requests.py index 4c69342e..ff21b7e2 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -3,7 +3,7 @@ import pytest from web_poet.page_inputs import ResponseData from web_poet.requests import ( - GenericRequest, + Request, perform_request, HttpClient, RequestBackendError, @@ -25,13 +25,13 @@ async def async_test(req): def test_generic_request(): - req = GenericRequest("url") + req = Request("url") assert req.url == "url" assert req.method == "GET" assert req.headers is None assert req.body is None - req = GenericRequest( + req = Request( "url", method="POST", headers={"User-Agent": "test agent"}, body=b"body" ) assert req.method == "POST" @@ -42,7 +42,7 @@ def test_generic_request(): @pytest.mark.asyncio async def test_perform_request(async_mock): - req = GenericRequest("url") + req = Request("url") with pytest.raises(RequestBackendError): await perform_request(req) @@ -56,16 +56,36 @@ async def test_perform_request(async_mock): @pytest.mark.asyncio -async def test_http_client(async_mock): +async def test_http_client_single_requests(async_mock): client = HttpClient(async_mock) assert client.request_downloader == async_mock - req_1 = GenericRequest("url-1") - req_2 = GenericRequest("url-2") + with mock.patch("web_poet.requests.Request") as mock_request: + response = await client.request("url") + response.url == "url" - # It should be able to accept arbitrary number of requests - client.request(req_1) - responses = await client.request(req_1, req_2) + response = await client.get("url-get") + response.url == "url-get" - assert responses[0].url == req_1.url - assert responses[1].url == req_2.url + response = await client.post("url-post") + response.url == "url-post" + + assert mock_request.call_args_list == [ + mock.call("url", "GET", None, None), + mock.call("url-get", "GET", None, None), + mock.call("url-post", "POST", None, None), + ] + + +@pytest.mark.asyncio +async def test_http_client_batch_requests(async_mock): + client = HttpClient(async_mock) + + requests = [ + Request("url-1"), + Request("url-get", method="GET"), + Request("url-post", method="POST"), + ] + responses = await client.batch_requests(*requests) + + assert all([isinstance(response, ResponseData) for response in responses]) diff --git a/web_poet/__init__.py b/web_poet/__init__.py index f143e1b4..918dea29 100644 --- a/web_poet/__init__.py +++ b/web_poet/__init__.py @@ -1,3 +1,3 @@ from .pages import WebPage, ItemPage, ItemWebPage, Injectable from .page_inputs import ResponseData -from .requests import request_backend_var, GenericRequest, HttpClient +from .requests import request_backend_var, Request, HttpClient diff --git a/web_poet/requests.py b/web_poet/requests.py index 12f6329c..6fe60c49 100644 --- a/web_poet/requests.py +++ b/web_poet/requests.py @@ -50,9 +50,13 @@ import attr +from web_poet.page_inputs import ResponseData + logger = logging.getLogger(__name__) +mapping = Dict[Union[str, ByteString], Any] + # Frameworks that wants to support additional requests in ``web-poet`` should # set the appropriate implementation for requesting data. request_backend_var: ContextVar = ContextVar("request_backend") @@ -63,16 +67,16 @@ class RequestBackendError(Exception): @attr.define -class GenericRequest: +class Request: """Represents a generic HTTP request.""" url: str method: str = "GET" - headers: Optional[Dict[Union[str, ByteString], Any]] = None + headers: Optional[mapping] = None body: Optional[str] = None -async def perform_request(request: GenericRequest): +async def perform_request(request: Request): logger.info(f"Requesting page: {request}") try: @@ -92,7 +96,25 @@ class HttpClient: def __init__(self, request_downloader=None): self.request_downloader = request_downloader or perform_request - async def request(self, *requests: List[GenericRequest]): + async def request( + self, + url: str, + method: str = "GET", + headers: Optional[mapping] = None, + body: Optional[str] = None, + ) -> ResponseData: + r = Request(url, method, headers, body) + return await self.request_downloader(r) + + async def get(self, url: str, headers: Optional[mapping] = None) -> ResponseData: + return await self.request(url=url, method="GET", headers=headers) + + async def post( + self, url: str, headers: Optional[mapping] = None, body: Optional[str] = None + ) -> ResponseData: + return await self.request(url=url, method="POST", headers=headers, body=body) + + async def batch_requests(self, *requests: List[Request]): coroutines = [self.request_downloader(r) for r in requests] responses = await asyncio.gather(*coroutines) return responses