New ItemPage

.github/workflows/test.yml

@@ -41,7 +41,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.10']
-        tox-job: ["mypy", "docs", "linters"]
+        tox-job: ["mypy", "docs", "linters", "types"]
 
     steps:
     - uses: actions/checkout@v2

pyproject.toml

@@ -1,3 +1,8 @@
 [tool.isort]
 profile = "black"
 multi_line_output = 3
+
+[tool.mypy]
+show_error_codes = true
+ignore_missing_imports = true
+no_warn_no_return = true

tests/test_fields.py

@@ -6,6 +6,7 @@
 
 from web_poet import (
     HttpResponse,
+    Injectable,
     ItemPage,
     field,
     item_from_fields,
@@ -21,7 +22,7 @@ class Item:
 
 
 @attrs.define
-class Page(ItemPage):
+class Page(ItemPage[Item]):
     response: HttpResponse
 
     @field
@@ -33,12 +34,9 @@ async def price(self):  # noqa: D102
         await asyncio.sleep(0.01)
         return "$123"
 
-    async def to_item(self):  # noqa: D102
-        return await item_from_fields(self, Item)
-
 
 @attrs.define
-class InvalidPage(ItemPage):
+class InvalidPage(ItemPage[Item]):
     response: HttpResponse
 
     @field
@@ -49,9 +47,6 @@ def name(self):  # noqa: D102
     def unknown_attribute(self):  # noqa: D102
         return "foo"
 
-    async def to_item(self):  # noqa: D102
-        return await item_from_fields(self, Item)
-
 
 EXAMPLE_RESPONSE = HttpResponse(
     "http://example.com",
@@ -136,13 +131,16 @@ async def to_item(self):
 
 
 def test_field_decorator_no_arguments():
-    class Page:
+    class Page(ItemPage):
         @field()
         def name(self):
             return "Name"
 
+        def to_item(self):
+            return item_from_fields_sync(self)
+
     page = Page()
-    assert item_from_fields_sync(page) == {"name": "Name"}
+    assert page.to_item() == {"name": "Name"}
 
 
 def test_field_cache_sync():
@@ -244,7 +242,7 @@ async def to_item(self) -> Item:
 
 def test_item_cls_fields():
     @attrs.define
-    class SyncPage(ItemPage):
+    class SyncPage(Injectable):
         response: HttpResponse
 
         @field
@@ -297,3 +295,32 @@ def to_item(self):
 
         assert fields["field2"].name == "field2"
         assert fields["field2"].meta is None
+
+
+def test_field_subclassing():
+    class Page(ItemPage):
+        @field
+        def field1(self):
+            return 1
+
+        @field
+        def field3(self):
+            return 1
+
+    assert list(get_fields_dict(Page)) == ["field1", "field3"]
+    assert get_fields_dict(Page)["field3"].meta is None
+
+    class Page2(Page):
+        @field
+        def field2(self):
+            return 1
+
+        @field(meta={"foo": "bar"})
+        def field3(self):
+            return 1
+
+    assert get_fields_dict(Page2)["field3"].meta == {"foo": "bar"}
+    assert list(get_fields_dict(Page2)) == ["field1", "field3", "field2"]
+
+    assert get_fields_dict(Page)["field3"].meta is None
+    assert list(get_fields_dict(Page)) == ["field1", "field3"]

tests/test_pages.py

@@ -1,22 +1,17 @@
+import attrs
 import pytest
 
-from web_poet.pages import ItemPage, ItemWebPage, is_injectable
+from web_poet import field
+from web_poet.pages import Injectable, ItemPage, ItemT, ItemWebPage, is_injectable
 
 
-def test_abstract_page_object():
-    with pytest.raises(TypeError) as exc:
-        ItemPage()
-    assert "Can't instantiate abstract class" in str(exc.value)
-
-
-def test_abstract_web_page_object():
-    with pytest.raises(TypeError) as exc:
-        ItemWebPage()
-    assert "Can't instantiate abstract class" in str(exc.value)
+@attrs.define
+class Item:
+    name: str
 
 
 def test_page_object():
-    class MyItemPage(ItemPage):
+    class MyItemPage(Injectable):
         def to_item(self) -> dict:
             return {
                 "foo": "bar",
@@ -30,7 +25,7 @@ def to_item(self) -> dict:
 
 def test_web_page_object(book_list_html_response):
     class MyWebPage(ItemWebPage):
-        def to_item(self) -> dict:
+        def to_item(self) -> dict:  # type: ignore
             return {
                 "url": self.url,
                 "title": self.css("title::text").get().strip(),
@@ -48,7 +43,7 @@ class MyClass:
         pass
 
     class MyItemPage(ItemPage):
-        def to_item(self) -> dict:
+        def to_item(self) -> dict:  # type: ignore
             return {
                 "foo": "bar",
             }
@@ -60,3 +55,86 @@ def to_item(self) -> dict:
     assert is_injectable(MyItemPage()) is False
     assert is_injectable(ItemPage) is True
     assert is_injectable(ItemWebPage) is True
+
+
+@pytest.mark.asyncio
+async def test_item_page_typed():
+    class MyPage(ItemPage[Item]):
+        @field
+        def name(self):
+            return "name"
+
+    page = MyPage()
+    assert page.item_cls is Item
+    item = await page.to_item()
+    assert isinstance(item, Item)
+    assert item == Item(name="name")
+
+
+@pytest.mark.asyncio
+async def test_item_page_typed_subclass():
+    class BasePage(ItemPage[ItemT]):
+        @field
+        def name(self):
+            return "name"
+
+    class Subclass(BasePage[Item]):
+        pass
+
+    page = BasePage()
+    assert page.item_cls is dict
+    assert (await page.to_item()) == {"name": "name"}
+
+    page2 = Subclass()
+    assert page2.item_cls is Item
+    assert (await page2.to_item()) == Item(name="name")
+
+
+@pytest.mark.asyncio
+async def test_item_page_change_item_type_extra_fields() -> None:
+    class BasePage(ItemPage[Item]):
+        @field
+        def name(self):
+            return "hello"
+
+    @attrs.define
+    class MyItem(Item):
+        price: float
+
+    class Subclass(BasePage, ItemPage[MyItem]):
+        @field
+        def price(self):
+            return 123
+
+    page = Subclass()
+    assert page.item_cls is MyItem
+    item = await page.to_item()
+    assert isinstance(item, MyItem)
+    assert item == MyItem(name="hello", price=123)
+
+
+@pytest.mark.asyncio
+async def test_item_page_change_item_type_remove_fields() -> None:
+    @attrs.define
+    class MyItem:
+        name: str
+        price: float
+
+    class BasePage(ItemPage[MyItem]):
+        @field
+        def name(self):
+            return "hello"
+
+        @field
+        def price(self):
+            return 123
+
+    # Item only contains "name", but not "price"
+    class Subclass(BasePage, ItemPage[Item], skip_nonitem_fields=True):
+        pass
+
+    page = Subclass()
+    assert page.item_cls is Item
+    item = await page.to_item()
+    assert isinstance(item, Item)
+    assert item == Item(name="hello")

tests_typing/test_fields.mypy-testing

@@ -0,0 +1,41 @@
+import pytest
+import attrs
+
+from web_poet import ItemPage, field, item_from_fields, item_from_fields_sync
+
+
+class Page(ItemPage):
+    @field
+    def name(self):
+        return "hello"
+
+
+@attrs.define
+class Item:
+    name: str
+
+
+@pytest.mark.mypy_testing
+async def test_item_from_fields() -> None:
+    page = Page()
+    item1 = await item_from_fields(page, item_cls=dict)
+    reveal_type(item1)  # R: builtins.dict[Any, Any]
+    item2 = await item_from_fields(page, item_cls=Item)
+    reveal_type(item2)  # R: __main__.Item
+
+
+@pytest.mark.mypy_testing
+def test_item_from_fields_sync() -> None:
+    page = Page()
+    item1 = item_from_fields_sync(page, item_cls=dict)
+    reveal_type(item1)  # R: builtins.dict[Any, Any]
+    item2 = item_from_fields_sync(page, item_cls=Item)
+    reveal_type(item2)  # R: __main__.Item
+
+
+@pytest.mark.mypy_testing
+@pytest.mark.xfail
+async def test_item_from_fields_default_item_cls() -> None:
+    page = Page()
+    item1 = await item_from_fields(page)
+    reveal_type(item1)  # R: builtins.dict[Any, Any]

tests_typing/test_item_page.mypy-testing

@@ -0,0 +1,90 @@
+import attr
+import pytest
+import attrs
+
+from web_poet import ItemPage, field
+from web_poet.pages import ItemT
+
+
+@attrs.define
+class Item:
+    name: str
+
+
+@pytest.mark.mypy_testing
+@pytest.mark.xfail
+async def test_item_page() -> None:
+    class MyPage(ItemPage):
+        @field
+        def name(self):
+            return "hello"
+
+    page = MyPage()
+    item = await page.to_item()
+    reveal_type(item)  # R: dict
+
+
+@pytest.mark.mypy_testing
+async def test_item_page_parametrized() -> None:
+    class MyPage(ItemPage[Item]):
+        @field
+        def name(self):
+            return "hello"
+
+    page = MyPage()
+    item = await page.to_item()
+    reveal_type(item)  # R: __main__.Item
+
+
+@pytest.mark.mypy_testing
+async def test_item_page_parametrized_subclass() -> None:
+    class BasePage(ItemPage[Item]):
+        @field
+        def name(self):
+            return "hello"
+
+
+    class Subclass(BasePage):
+        pass
+
+    page = Subclass()
+    item = await page.to_item()
+    reveal_type(item)  # R: __main__.Item
+
+
+@pytest.mark.mypy_testing
+async def test_item_page_subclass_parametrized() -> None:
+    class BasePage(ItemPage[ItemT]):
+        @field
+        def name(self):
+            return "hello"
+
+    class Subclass(BasePage[Item]):
+        pass
+
+    page = Subclass()
+    item = await page.to_item()
+    reveal_type(item)  # R: __main__.Item
+
+
+@pytest.mark.mypy_testing
+@pytest.mark.xfail
+async def test_item_page_change_type() -> None:
+
+    class BasePage(ItemPage[Item]):
+        @field
+        def name(self):
+            return "hello"
+
+    @attr.define
+    class MyItem(Item):
+        price: float
+
+    class Subclass(BasePage, ItemPage[MyItem]):
+        @field
+        def price(self):
+            return 123
+
+    page = Subclass()
+    item = await page.to_item()
+    reveal_type(item)  # R: MyItem