Adding pub date check to standard run

spider/config.py

@@ -22,7 +22,8 @@
   "fetch_abstracts": True, # Check for any Rxivist papers missing an abstract and fill it in (Papers don't have an abstract when first crawled)
   "fetch_crossref": False, # Update daily Crossref stats
   "refresh_stats": False, # Look for articles with outdated download info and re-crawl them
-  "fetch_pubstatus": True # Check for whether a paper has been published during stat refresh
+  "fetch_pubstatus": True, # Check for whether a paper has been published during stat refresh
+  "fetch_pubdates": True # Check for publication dates for any papers that have been published
 }
 
 perform_ranks = {

spider/spider.py

@@ -953,7 +953,12 @@ def full_run(spider):
   if config.crawl["fetch_crossref"] is not False:
     spider.pull_todays_crossref_data()
   else:
-    spider.log.record("Skipping call to fetch Crossref data: disabled in configuration file.", 'debug')
+    spider.log.record("Skipping call to fetch Crossref Twitter data: disabled in configuration file.", 'debug')
+
+  if config.crawl["fetch_pubdates"] is not False:
+    get_publication_dates(spider)
+  else:
+    spider.log.record("Skipping call to fetch Crossref publication data: disabled in configuration file.", 'debug')
 
   spider.calculate_vectors()
 
@@ -1023,7 +1028,7 @@ def get_publication_dates(spider):
           break
         done += 1
         if config.polite:
-          time.sleep(2)
+          time.sleep(3)
         article_id = article[0]
         doi = article[1]
         spider.log.record(f"Checking DOI {doi}", 'debug')