Skip to content

Commit

Permalink
Test that meta (empty page params and arbitrary keys) does not impact…
Browse files Browse the repository at this point in the history
… fingerprinting
  • Loading branch information
Gallaecio committed Nov 15, 2023
1 parent 23b7c33 commit e8f0322
Showing 1 changed file with 30 additions and 6 deletions.
36 changes: 30 additions & 6 deletions tests/test_request_fingerprinter.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,12 +334,9 @@ async def parse_b(self, response, web: WebPage, http_response: HttpResponse):


def test_page_params(caplog):
class TestSpider(Spider):
name = "test_spider"

Unserializable = object()

crawler = get_crawler(spider_cls=TestSpider)
crawler = get_crawler()
fingerprinter = crawler.request_fingerprinter

request1 = Request("https://toscrape.com")
Expand All @@ -352,17 +349,44 @@ class TestSpider(Spider):
fingerprint3 = fingerprinter.fingerprint(request3)

request4 = Request(
"https://toscrape.com", meta={"page_params": {"a": "2"}, "foo": "bar"}
)
fingerprint4 = fingerprinter.fingerprint(request4)

request5 = Request(
"https://toscrape.com", meta={"page_params": {"a": Unserializable}}
)
assert "Cannot serialize page params" not in caplog.text
caplog.clear()
fingerprint4 = fingerprinter.fingerprint(request4)
fingerprint5 = fingerprinter.fingerprint(request5)
assert "Cannot serialize page params" in caplog.text

assert fingerprint1 != fingerprint2
assert fingerprint1 != fingerprint3
assert fingerprint2 != fingerprint3
assert fingerprint1 == fingerprint4
assert fingerprint3 == fingerprint4
assert fingerprint1 == fingerprint5


@pytest.mark.parametrize(
"meta",
(
{},
{"page_params": None},
{"page_params": {}},
{"foo": "bar"},
{"foo": "bar", "page_params": None},
{"foo": "bar", "page_params": {}},
),
)
def test_meta(meta):
crawler = get_crawler()
fingerprinter = crawler.request_fingerprinter
request1 = Request("https://toscrape.com")
fingerprint1 = fingerprinter.fingerprint(request1)
request2 = Request("https://toscrape.com", meta=meta)
fingerprint2 = fingerprinter.fingerprint(request2)
assert fingerprint1 == fingerprint2


@pytest.mark.skipif(
Expand Down

0 comments on commit e8f0322

Please sign in to comment.