From 54aa80913cc66d99e4611fa58b37b54742e44bbc Mon Sep 17 00:00:00 2001 From: isaac Date: Sun, 17 Oct 2021 19:34:16 +0200 Subject: [PATCH] reset flag can now also be str of spider name in addition to bool --- scrapy_deltafetch/middleware.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy_deltafetch/middleware.py b/scrapy_deltafetch/middleware.py index 9f6f1b4..02c4f16 100644 --- a/scrapy_deltafetch/middleware.py +++ b/scrapy_deltafetch/middleware.py @@ -37,7 +37,13 @@ def from_crawler(cls, crawler): if not s.getbool('DELTAFETCH_ENABLED'): raise NotConfigured dir = data_path(s.get('DELTAFETCH_DIR', 'deltafetch')) - reset = s.getbool('DELTAFETCH_RESET') + + # DELTAFETCH_RESET can be bool (for all spiders) or str of specific spider name + try: + reset = s.getbool('DELTAFETCH_RESET') + except ValueError: + reset = s.get('DELTAFETCH_RESET') + o = cls(dir, reset, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -49,7 +55,7 @@ def spider_opened(self, spider): # TODO may be tricky, as there may be different paths on systems dbpath = os.path.join(self.dir, '%s.db' % spider.name) reset = self.reset or getattr(spider, 'deltafetch_reset', False) - flag = 'n' if reset else 'c' + flag = 'n' if reset is True or reset == spider.name else 'c' try: self.db = dbm.open(dbpath, flag=flag) except Exception: