add tests for additional request components

tests/test_page_inputs.py

@@ -2,6 +2,12 @@
 
 
 def test_html_response():
-    response = ResponseData('url', 'content')
-    assert response.url == 'url'
-    assert response.html == 'content'
+    response = ResponseData("url", "content")
+    assert response.url == "url"
+    assert response.html == "content"
+    assert response.status is None
+    assert response.headers is None
+
+    response = ResponseData("url", "content", 200, {"User-Agent": "test agent"})
+    assert response.status == 200
+    assert response.headers["User-Agent"] == "test agent"

tests/test_requests.py

@@ -0,0 +1,71 @@
+from unittest import mock
+
+import pytest
+from web_poet.page_inputs import ResponseData
+from web_poet.requests import (
+    GenericRequest,
+    perform_request,
+    HttpClient,
+    RequestBackendError,
+    request_backend_var,
+)
+
+
+@pytest.fixture
+def async_mock():
+    """workaround since python 3.7 doesn't ship with asyncmock."""
+
+    async def async_test(req):
+        return ResponseData(req.url, req.body)
+
+    mock.MagicMock.__await__ = lambda x: async_test().__await__()
+
+    return async_test
+
+
+def test_generic_request():
+
+    req = GenericRequest("url")
+    assert req.url == "url"
+    assert req.method == "GET"
+    assert req.headers is None
+    assert req.body is None
+
+    req = GenericRequest(
+        "url", method="POST", headers={"User-Agent": "test agent"}, body=b"body"
+    )
+    assert req.method == "POST"
+    assert req.headers == {"User-Agent": "test agent"}
+    assert req.body == b"body"
+
+
+@pytest.mark.asyncio
+async def test_perform_request(async_mock):
+
+    req = GenericRequest("url")
+
+    with pytest.raises(RequestBackendError):
+        await perform_request(req)
+
+    request_backend_var.set(async_mock)
+    response = await perform_request(req)
+
+    # The async downloader implementation should return the ResponseData
+    assert response.url == req.url
+    assert type(response) == ResponseData
+
+
+@pytest.mark.asyncio
+async def test_http_client(async_mock):
+    client = HttpClient(async_mock)
+    assert client.request_downloader == async_mock
+
+    req_1 = GenericRequest("url-1")
+    req_2 = GenericRequest("url-2")
+
+    # It should be able to accept arbitrary number of requests
+    client.request(req_1)
+    responses = await client.request(req_1, req_2)
+
+    assert responses[0].url == req_1.url
+    assert responses[1].url == req_2.url

tox.ini

@@ -1,10 +1,14 @@
 [tox]
 envlist = py37,py38,py39,py310,mypy,docs
 
+[pytest]
+asyncio_mode = strict
+
 [testenv]
 deps =
     pytest
     pytest-cov
+    pytest-asyncio
 
 commands =
     py.test \

web_poet/__init__.py

@@ -1,3 +1,3 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
 from .page_inputs import ResponseData
-from .requests import perform_request, request_backend_var, GenericRequest
+from .requests import request_backend_var, GenericRequest, HttpClient

web_poet/requests.py

@@ -46,7 +46,7 @@
 import asyncio
 import logging
 from contextvars import ContextVar
-from typing import Optional, List
+from typing import Optional, List, Dict, ByteString, Any, Union
 
 import attr
 
@@ -68,7 +68,7 @@ class GenericRequest:
 
     url: str
     method: str = "GET"
-    headers: Optional[str] = None
+    headers: Optional[Dict[Union[str, ByteString], Any]] = None
     body: Optional[str] = None