Merge pull request #161 from proway2/po-stats

Injector stats

docs/stats.rst

@@ -6,3 +6,11 @@ Stats
 
 scrapy-poet tracks :external+web-poet:ref:`web-poet stats <stats>` as part of
 :ref:`Scrapy stats <topics-stats>`, prefixed with ``poet/stats/``.
+
+Injector stats
+==============
+
+The injector produces some stats also. These are:
+
+* cache stats have the ``poet/cache`` prefix.
+* injected dependencies stats have the ``poet/injector`` prefix.

requirements-dev.txt

@@ -1,2 +1,3 @@
 pre-commit
 pytest
+pytest-twisted

scrapy_poet/injection.py

@@ -3,7 +3,7 @@
 import os
 import pprint
 import warnings
-from typing import Any, Callable, Dict, List, Mapping, Optional, Set
+from typing import Any, Callable, Dict, List, Mapping, Optional, Set, cast
 
 import andi
 from andi.typeutils import issubclass_safe
@@ -20,6 +20,7 @@
 from web_poet.page_inputs.http import request_fingerprint
 from web_poet.pages import is_injectable
 from web_poet.serialization.api import deserialize_leaf, load_class, serialize
+from web_poet.utils import get_fq_class_name
 
 from scrapy_poet.api import _CALLBACK_FOR_MARKER, DummyResponse
 from scrapy_poet.cache import SerializedDataCache
@@ -161,6 +162,8 @@ def build_instances(self, request: Request, response: Response, plan: andi.Plan)
         for cls, kwargs_spec in plan.dependencies:
             if cls not in instances.keys():
                 instances[cls] = cls(**kwargs_spec.kwargs(instances))
+                cls_fqn = get_fq_class_name(cast(type, cls))
+                self.crawler.stats.inc_value(f"poet/injector/{cls_fqn}")
 
         return instances
 
@@ -198,9 +201,9 @@ def build_instances_from_providers(
                 try:
                     data = self.cache[fingerprint].items()
                 except KeyError:
-                    self.crawler.stats.inc_value("scrapy-poet/cache/miss")
+                    self.crawler.stats.inc_value("poet/cache/miss")
                 else:
-                    self.crawler.stats.inc_value("scrapy-poet/cache/hit")
+                    self.crawler.stats.inc_value("poet/cache/hit")
                     if isinstance(data, Exception):
                         raise data
                     objs = [
@@ -228,7 +231,7 @@ def build_instances_from_providers(
                     if self.cache and self.caching_errors:
                         # Save errors in the cache
                         self.cache[fingerprint] = e
-                        self.crawler.stats.inc_value("scrapy-poet/cache/firsthand")
+                        self.crawler.stats.inc_value("poet/cache/firsthand")
                     raise
 
             objs_by_type: Dict[Callable, Any] = {type(obj): obj for obj in objs}
@@ -244,7 +247,7 @@ def build_instances_from_providers(
             if self.cache and not cache_hit:
                 # Save the results in the cache
                 self.cache[fingerprint] = serialize(objs)
-                self.crawler.stats.inc_value("scrapy-poet/cache/firsthand")
+                self.crawler.stats.inc_value("poet/cache/firsthand")
 
         return instances
 

tests/test_injection.py

@@ -305,6 +305,82 @@ def to_item(self):
         return item
 
 
+@attr.s(auto_attribs=True)
+class TestItem:
+    foo: int
+    bar: str
+
+
+class TestItemPage(ItemPage[TestItem]):
+    async def to_item(self):
+        return TestItem(foo=1, bar="bar")
+
+
+class TestInjectorStats:
+    @pytest.mark.parametrize(
+        "cb_args, expected",
+        (
+            (
+                {"price_po": PricePO, "rate_po": EurDollarRate},
+                {
+                    "tests.test_injection.PricePO",
+                    "tests.test_injection.EurDollarRate",
+                    "tests.test_injection.Html",
+                },
+            ),
+            (
+                {"price_po": PriceInDollarsPO},
+                {
+                    "tests.test_injection.PricePO",
+                    "tests.test_injection.PriceInDollarsPO",
+                    "tests.test_injection.Html",
+                    "tests.test_injection.EurDollarRate",
+                },
+            ),
+            (
+                {},
+                set(),
+            ),
+            (
+                {"item": TestItem},
+                set(),  # there must be no stats as ItemProvider is not enabled
+            ),
+        ),
+    )
+    @inlineCallbacks
+    def test_stats(self, cb_args, expected, injector):
+        def callback_factory():
+            args = ", ".join([f"{k}: {v.__name__}" for k, v in cb_args.items()])
+            exec(f"def callback(response: DummyResponse, {args}): pass")
+            return locals().get("callback")
+
+        callback = callback_factory()
+        response = get_response_for_testing(callback)
+        _ = yield from injector.build_callback_dependencies(response.request, response)
+        prefix = "poet/injector/"
+        poet_stats = {
+            name.replace(prefix, ""): value
+            for name, value in injector.crawler.stats.get_stats().items()
+            if name.startswith(prefix)
+        }
+        assert set(poet_stats) == expected
+
+    @inlineCallbacks
+    def test_po_provided_via_item(self, injector):
+        rules = [ApplyRule(Patterns(include=()), use=TestItemPage, to_return=TestItem)]
+        registry = RulesRegistry(rules=rules)
+        providers = {"scrapy_poet.page_input_providers.ItemProvider": 10}
+        injector = get_injector_for_testing(providers, registry=registry)
+
+        def callback(response: DummyResponse, item: TestItem):
+            pass
+
+        response = get_response_for_testing(callback)
+        _ = yield from injector.build_callback_dependencies(response.request, response)
+        key = "poet/injector/tests.test_injection.TestItemPage"
+        assert key in set(injector.crawler.stats.get_stats())
+
+
 class TestInjectorOverrides:
     @pytest.mark.parametrize("override_should_happen", [True, False])
     @inlineCallbacks