diff --git a/tests/test_page_inputs.py b/tests/test_page_inputs.py index 8a74589c..7988e69f 100644 --- a/tests/test_page_inputs.py +++ b/tests/test_page_inputs.py @@ -2,6 +2,12 @@ def test_html_response(): - response = ResponseData('url', 'content') - assert response.url == 'url' - assert response.html == 'content' + response = ResponseData("url", "content") + assert response.url == "url" + assert response.html == "content" + assert response.status is None + assert response.headers is None + + response = ResponseData("url", "content", 200, {"User-Agent": "test agent"}) + assert response.status == 200 + assert response.headers["User-Agent"] == "test agent" diff --git a/tests/test_requests.py b/tests/test_requests.py new file mode 100644 index 00000000..4c69342e --- /dev/null +++ b/tests/test_requests.py @@ -0,0 +1,71 @@ +from unittest import mock + +import pytest +from web_poet.page_inputs import ResponseData +from web_poet.requests import ( + GenericRequest, + perform_request, + HttpClient, + RequestBackendError, + request_backend_var, +) + + +@pytest.fixture +def async_mock(): + """workaround since python 3.7 doesn't ship with asyncmock.""" + + async def async_test(req): + return ResponseData(req.url, req.body) + + mock.MagicMock.__await__ = lambda x: async_test().__await__() + + return async_test + + +def test_generic_request(): + + req = GenericRequest("url") + assert req.url == "url" + assert req.method == "GET" + assert req.headers is None + assert req.body is None + + req = GenericRequest( + "url", method="POST", headers={"User-Agent": "test agent"}, body=b"body" + ) + assert req.method == "POST" + assert req.headers == {"User-Agent": "test agent"} + assert req.body == b"body" + + +@pytest.mark.asyncio +async def test_perform_request(async_mock): + + req = GenericRequest("url") + + with pytest.raises(RequestBackendError): + await perform_request(req) + + request_backend_var.set(async_mock) + response = await perform_request(req) + + # The async downloader implementation should return the ResponseData + assert response.url == req.url + assert type(response) == ResponseData + + +@pytest.mark.asyncio +async def test_http_client(async_mock): + client = HttpClient(async_mock) + assert client.request_downloader == async_mock + + req_1 = GenericRequest("url-1") + req_2 = GenericRequest("url-2") + + # It should be able to accept arbitrary number of requests + client.request(req_1) + responses = await client.request(req_1, req_2) + + assert responses[0].url == req_1.url + assert responses[1].url == req_2.url diff --git a/tox.ini b/tox.ini index ea67701a..87df20fe 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,14 @@ [tox] envlist = py37,py38,py39,py310,mypy,docs +[pytest] +asyncio_mode = strict + [testenv] deps = pytest pytest-cov + pytest-asyncio commands = py.test \ diff --git a/web_poet/__init__.py b/web_poet/__init__.py index 68715aed..f143e1b4 100644 --- a/web_poet/__init__.py +++ b/web_poet/__init__.py @@ -1,3 +1,3 @@ from .pages import WebPage, ItemPage, ItemWebPage, Injectable from .page_inputs import ResponseData -from .requests import perform_request, request_backend_var, GenericRequest +from .requests import request_backend_var, GenericRequest, HttpClient diff --git a/web_poet/requests.py b/web_poet/requests.py index 8d53de3c..12f6329c 100644 --- a/web_poet/requests.py +++ b/web_poet/requests.py @@ -46,7 +46,7 @@ import asyncio import logging from contextvars import ContextVar -from typing import Optional, List +from typing import Optional, List, Dict, ByteString, Any, Union import attr @@ -68,7 +68,7 @@ class GenericRequest: url: str method: str = "GET" - headers: Optional[str] = None + headers: Optional[Dict[Union[str, ByteString], Any]] = None body: Optional[str] = None