Add Scraper Mode

src/config.ts

@@ -1,4 +1,7 @@
 import {
+    ScraperMode,
+    DEFAULT_RESCRAPE_BLOCKS,
+    DEFAULT_SCRAPER_MODE,
     DEFAULT_BASE_GITHUB_LOGO_URL,
     DEFAULT_BLOCKS_REORG_CHECK_INCREMENT,
     DEFAULT_BLOCK_FINALITY_THRESHOLD,
@@ -69,8 +72,6 @@ interface BridgeContract {
     startingBlock: number;
 }
 
-export const RESCRAPE_BLOCKS = 10; //TODO: delete me
-
 const bridgeContracts = [
     { contract: '0x1c29670f7a77f1052d30813a0a4f632c78a02610', startingBlock: 9613431 },
     { contract: '0x991c745401d5b5e469b8c3e2cb02c748f08754f1', startingBlock: 9613441 },
@@ -111,6 +112,14 @@ export const POSTGRES_URI = process.env.POSTGRES_URI || DEFAULT_LOCAL_POSTGRES_U
 export const SHOULD_SYNCHRONIZE = process.env.SHOULD_SYNCHRONIZE === 'true';
 
 export const ENABLE_PROMETHEUS_METRICS = getBoolConfig('ENABLE_PROMETHEUS_METRICS', DEFAULT_ENABLE_PROMETHEUS_METRICS);
+export const SCRAPER_MODE: ScraperMode =
+    process.env.SCRAPER_MODE === undefined
+        ? DEFAULT_SCRAPER_MODE
+        : process.env.SCRAPER_MODE === 'BLOCKS'
+        ? 'BLOCKS'
+        : process.env.SCRAPER_MODE === 'EVENTS'
+        ? 'EVENTS'
+        : throwError('Wrong SCRAPER_MODE');
 export const METRICS_PATH = process.env.METRICS_PATH || DEFAULT_METRICS_PATH;
 
 export const PROMETHEUS_PORT = getIntConfig('PROMETHEUS_PORT', DEFAULT_PROMETHEUS_PORT);
@@ -374,6 +383,15 @@ export const FEAT_TOKENS_FROM_TRANSFERS = getBoolConfig(
     DEFAULT_FEAT_TOKENS_FROM_TRANSFERS,
 );
 
+export const TOKENS_FROM_TRANSFERS_START_BLOCK = getIntConfig('TOKENS_FROM_TRANSFERS_START_BLOCK', -1);
+
+validateStartBlock(
+    'TOKENS_FROM_TRANSFERS_START_BLOCK',
+    TOKENS_FROM_TRANSFERS_START_BLOCK,
+    "FEAT_TOKENS_FROM_TRANSFERS && SCRAPER_MODE !== 'BLOCKS'",
+    FEAT_TOKENS_FROM_TRANSFERS && SCRAPER_MODE !== 'BLOCKS',
+);
+
 export const FEAT_META_TRANSACTION_EXECUTED_EVENT = getBoolConfig(
     'FEAT_META_TRANSACTION_EXECUTED_EVENT',
     DEFAULT_FEAT_META_TRANSACTION_EXECUTED_EVENT,
@@ -450,6 +468,8 @@ validateStartBlock(
     FEAT_SOCKET_BRIDGE_EVENT,
 );
 
+export const RESCRAPE_BLOCKS = getIntConfig('RESCRAPE_BLOCKS', DEFAULT_RESCRAPE_BLOCKS);
+
 export const BLOCKS_REORG_CHECK_INCREMENT = getIntConfig(
     'BLOCKS_REORG_CHECK_INCREMENT',
     DEFAULT_BLOCKS_REORG_CHECK_INCREMENT,

src/constants.ts

@@ -1,12 +1,16 @@
+export type ScraperMode = 'BLOCKS' | 'EVENTS';
+
 export * from './abis';
 
+export const DEFAULT_SCRAPER_MODE: ScraperMode = 'EVENTS';
 export const DEFAULT_LOCAL_POSTGRES_URI = 'postgresql://user:password@localhost/events';
 export const DEFAULT_MAX_BLOCKS_REORG = 35;
 export const DEFAULT_BLOCKS_REORG_CHECK_INCREMENT = 35;
 export const DEFAULT_MAX_BLOCKS_TO_PULL = 120;
 export const DEFAULT_MAX_BLOCKS_TO_SEARCH = 120;
 export const DEFAULT_MAX_TX_TO_PULL = 1000;
 export const DEFAULT_BLOCK_FINALITY_THRESHOLD = 10;
+export const DEFAULT_RESCRAPE_BLOCKS = 0;
 export const DEFAULT_MINUTES_BETWEEN_RUNS = 3;
 export const DEFAULT_STAKING_POOLS_JSON_URL =
     'https://raw.githubusercontent.com/0xProject/0x-staking-pool-registry/master/staking_pools.json';

src/scripts/pull_and_save_tokens_from_transfers.ts

@@ -1,5 +1,5 @@
 import { EVM_RPC_URL, MAX_BLOCKS_TO_SEARCH, TOKENS_FROM_TRANSFERS_START_BLOCK } from '../config';
-import { TOKEN_TRANSFER_EVENT_TOPIC } from '../constants';
+import { TRANSFER_EVENT_TOPIC_0 } from '../constants';
 import { LogPullInfo, Web3Source } from '../data_sources/events/web3';
 import { logger } from '../utils/logger';
 import { SCRIPT_RUN_DURATION, SCAN_START_BLOCK, SCAN_END_BLOCK } from '../utils/metrics';
@@ -60,7 +60,7 @@ export class TokensFromTransfersScraper {
             address: 'nofilter',
             fromBlock: startBlockNumber,
             toBlock: endBlockNumber,
-            topics: TOKEN_TRANSFER_EVENT_TOPIC,
+            topics: [TRANSFER_EVENT_TOPIC_0],
         };
 
         const rawLogsArray = await web3Source.getBatchLogInfoForContractsAsync([logPullInfo]);