Event Scrapers API v2 - Save all Txns and Transfer events

docker-compose.yml

@@ -35,6 +35,7 @@ services:
             #KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}'
             SCRAPER_MODE: 'BLOCKS'
             EP_DEPLOYMENT_BLOCK: 10247094
+            SETTLER_DEPLOYMENT_BLOCK: 19582253
             MAX_BLOCKS_TO_SEARCH: 1000
             MAX_BLOCKS_TO_PULL: 100
             MAX_TX_TO_PULL: 1000
@@ -78,6 +79,7 @@ services:
             FEAT_SOCKET_BRIDGE_EVENT: "true"
             SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5"
             SOCKET_BRIDGE_EVENT_START_BLOCK: 18473454
+            FEAT_ERC20_TRANSFER_ALL: "true"
 
     event-pipeline-bsc:
         depends_on:

src/abis.ts

@@ -1426,3 +1426,29 @@ export const STAKING_REWARDS_PAID_ABI = {
     name: 'RewardsPaid',
     type: 'event',
 };
+
+export const ERC20_TRANSFER_ABI = {
+    anonymous: false,
+    inputs: [
+        {
+            indexed: true,
+            internalType: 'address',
+            name: 'from',
+            type: 'address',
+        },
+        {
+            indexed: true,
+            internalType: 'address',
+            name: 'to',
+            type: 'address',
+        },
+        {
+            indexed: false,
+            internalType: 'uint256',
+            name: 'value',
+            type: 'uint256',
+        },
+    ],
+    name: 'Transfer',
+    type: 'event',
+};

src/config.ts

@@ -45,6 +45,7 @@ import {
     DEFAULT_STAKING_POOLS_METADATA_JSON_URL,
     DEFAULT_BLOCK_RECEIPTS_MODE,
     BLOCK_RECEIPTS_MODE_ENDPOINT,
+    DEFAULT_FEAT_ERC20_TRANSFER_ALL,
 } from './constants';
 import { logger } from './utils';
 
@@ -105,6 +106,10 @@ export const EP_DEPLOYMENT_BLOCK = process.env.EP_DEPLOYMENT_BLOCK
     ? parseInt(process.env.EP_DEPLOYMENT_BLOCK, 10)
     : throwError(`Must specify valid EP_DEPLOYMENT_BLOCK. Got: ${process.env.EP_DEPLOYMENT_BLOCK}`);
 
+export const SETTLER_DEPLOYMENT_BLOCK = process.env.SETTLER_DEPLOYMENT_BLOCK
+    ? parseInt(process.env.SETTLER_DEPLOYMENT_BLOCK, 10)
+    : throwError(`Must specify valid SETTLER_DEPLOYMENT_BLOCK. Got: ${process.env.SETTLER_DEPLOYMENT_BLOCK}`);
+
 export const SCHEMA = process.env.SCHEMA
     ? process.env.SCHEMA
     : throwError(`Must specify valid SCHEMA. Got: ${process.env.SCHEMA}`);
@@ -197,6 +202,7 @@ if (STAKING_PROXY_DEPLOYMENT_TRANSACTION === null && FEAT_STAKING) {
 export const FIRST_SEARCH_BLOCK = Math.min(
     EP_DEPLOYMENT_BLOCK,
     STAKING_DEPLOYMENT_BLOCK === -1 ? Infinity : STAKING_DEPLOYMENT_BLOCK,
+    SETTLER_DEPLOYMENT_BLOCK === -1 ? Infinity : SETTLER_DEPLOYMENT_BLOCK,
 );
 
 export const FEAT_TRANSFORMED_ERC20_EVENT = getBoolConfig(
@@ -482,6 +488,8 @@ export const BLOCKS_REORG_CHECK_INCREMENT = getIntConfig(
     DEFAULT_BLOCKS_REORG_CHECK_INCREMENT,
 );
 
+export const FEAT_ERC20_TRANSFER_ALL = getBoolConfig('FEAT_ERC20_TRANSFER_ALL', DEFAULT_FEAT_ERC20_TRANSFER_ALL);
+
 function getBoolConfig(env: string, defaultValue: boolean): boolean {
     if (Object.prototype.hasOwnProperty.call(process.env, env)) {
         return process.env[env] === 'true';

src/constants.ts

@@ -49,6 +49,7 @@ export const DEFAULT_ENABLE_PROMETHEUS_METRICS = false;
 export const DEFAULT_FEAT_ONCHAIN_GOVERNANCE = false;
 export const DEFAULT_FEAT_WRAP_UNWRAP_NATIVE_EVENT = false;
 export const DEFAULT_FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT = false;
+export const DEFAULT_FEAT_ERC20_TRANSFER_ALL = false;
 
 export const BRIDGEFILL_EVENT_TOPIC = ['0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8'];
 export const TRANSFORMEDERC20_EVENT_TOPIC = ['0x0f6672f78a59ba8e5e5b5d38df3ebc67f3c792e2c9259b8d97d7f00dd78ba1b3'];
@@ -60,6 +61,7 @@ export const DEFAULT_STAKING_PROXY_ADDRESS = '0xa26e80e7dea86279c6d778d702cc413e
 export const V4_CANCEL_EVENT_TOPIC = ['0xa6eb7cdc219e1518ced964e9a34e61d68a94e4f1569db3e84256ba981ba52753'];
 export const EXPIRED_RFQ_ORDER_EVENT_TOPIC = ['0xd9ee00a67daf7d99c37893015dc900862c9a02650ef2d318697e502e5fb8bbe2'];
 export const OTC_ORDER_FILLED_EVENT_TOPIC = ['0xac75f773e3a92f1a02b12134d65e1f47f8a14eabe4eaf1e24624918e6a8b269f'];
+export const ERC20_TRANSFER_EVENT_TOPIC = ['0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'];
 
 export const UNISWAP_V2_PAIR_CREATED_TOPIC = ['0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9'];
 export const UNISWAP_V2_SYNC_TOPIC = ['0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1'];

src/entities/erc20_transfer_event.ts

@@ -0,0 +1,16 @@
+import { bigNumberTransformer } from '../transformers';
+import { Event } from './event';
+import { BigNumber } from '@0x/utils';
+import { Column, Entity } from 'typeorm';
+
+// These events come directly from the token's ERC20 contract and are fired whenever
+// an ERC20 is transferred between two wallets.
+@Entity({ name: 'erc20_transfer_events' })
+export class ERC20TransferEvent extends Event {
+    @Column({ name: 'from', type: 'varchar' })
+    public from!: string;
+    @Column({ name: 'to', type: 'varchar' })
+    public to!: string | null;
+    @Column({ name: 'value', type: 'numeric', transformer: bigNumberTransformer })
+    public value!: BigNumber;
+}

src/entities/index.ts

@@ -50,3 +50,4 @@ export { V4CancelEvent } from './v4_cancel_event';
 export { V4LimitOrderFilledEvent } from './v4_limit_order_filled_event';
 export { V4RfqOrderFilledEvent } from './v4_rfq_order_filled_event';
 export { WrapNativeEvent } from './wrap_native_event';
+export { ERC20TransferEvent } from './erc20_transfer_event';

src/events.ts

@@ -2,6 +2,7 @@ import {
     EP_ADDRESS,
     FEAT_STAKING,
     EP_DEPLOYMENT_BLOCK,
+    SETTLER_DEPLOYMENT_BLOCK,
     FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET,
     FEAT_LIMIT_ORDERS,
     FEAT_META_TRANSACTION_EXECUTED_EVENT,
@@ -47,6 +48,7 @@ import {
     V4_NATIVE_FILL_START_BLOCK,
     WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS,
     WRAP_UNWRAP_NATIVE_START_BLOCK,
+    FEAT_ERC20_TRANSFER_ALL,
 } from './config';
 import {
     BRIDGEFILL_EVENT_TOPIC,
@@ -99,6 +101,7 @@ import {
     STAKING_EPOCH_ENDED_TOPIC,
     STAKING_EPOCH_FINALIZED_TOPIC,
     STAKING_REWARDS_PAID_TOPIC,
+    ERC20_TRANSFER_EVENT_TOPIC,
 } from './constants';
 import { Web3Source } from './data_sources/events/web3';
 import {
@@ -144,6 +147,7 @@ import {
     StakingPoolEarnedRewardsInEpochEvent,
     TransactionExecutionEvent,
     UnstakeEvent,
+    ERC20TransferEvent,
 } from './entities';
 import {
     filterSocketBridgeEventsGetContext,
@@ -189,6 +193,7 @@ import {
     parseV4RfqOrderFilledEvent,
     parseWrapNativeEvent,
     parseWrapNativeTransferEvent,
+    parseERC20TransferEvent,
 } from './parsers';
 import {
     parseEpochEndedEvent,
@@ -762,6 +767,17 @@ export const eventScrperProps: EventScraperProps[] = [
         startBlock: V3_DEPLOYMENT_BLOCK,
         parser: parseRewardsPaidEvent,
     },
+    {
+        enabled: FEAT_ERC20_TRANSFER_ALL,
+        name: 'ERC20TransferEvent',
+        tType: ERC20TransferEvent,
+        table: 'erc20_transfer_events',
+        topics: ERC20_TRANSFER_EVENT_TOPIC,
+        contractAddress: null,
+        startBlock: FLASHWALLET_DEPLOYMENT_BLOCK,
+        parser: parseERC20TransferEvent,
+        filterFunction: filterNulls,
+    },
 ];
 
 for (const payment_recipient of POLYGON_RFQM_PAYMENTS_ADDRESSES) {

src/ormconfig.ts

@@ -51,6 +51,7 @@ import {
     V4LimitOrderFilledEvent,
     V4RfqOrderFilledEvent,
     WrapNativeEvent,
+    ERC20TransferEvent,
 } from './entities';
 import { ConnectionOptions } from 'typeorm';
 
@@ -106,6 +107,7 @@ const entities = [
     V4LimitOrderFilledEvent,
     V4RfqOrderFilledEvent,
     WrapNativeEvent,
+    ERC20TransferEvent,
 ];
 
 const config: ConnectionOptions = {

src/parsers/events/erc20_transfer_events.ts

@@ -0,0 +1,29 @@
+import { ERC20_TRANSFER_ABI } from '../../constants';
+import { ERC20TransferEvent } from '../../entities';
+import { parseEvent } from './parse_event';
+import { BigNumber } from '@0x/utils';
+import { LogEntry } from 'ethereum-types';
+
+const abiCoder = require('web3-eth-abi');
+
+const Web3Utils = require('web3-utils');
+
+export function parseERC20TransferEvent(eventLog: LogEntry): ERC20TransferEvent | null {
+    const eRC20TransferEvent = new ERC20TransferEvent();
+
+    parseEvent(eventLog, eRC20TransferEvent);
+
+    // ERC20 Transfers have 3 topics and 32 bytes of data
+    // ERC721 Transfers have 4 topics and no data
+    if (eventLog.topics.length !== 3) {
+        return null;
+    }
+
+    const decodedLog = abiCoder.decodeLog(ERC20_TRANSFER_ABI.inputs, eventLog.data, eventLog.topics.slice(1));
+
+    eRC20TransferEvent.from = decodedLog.from.toLowerCase();
+    eRC20TransferEvent.to = decodedLog.to.toLowerCase();
+    eRC20TransferEvent.value = new BigNumber(decodedLog.value);
+
+    return eRC20TransferEvent;
+}

src/parsers/index.ts

@@ -46,3 +46,4 @@ export {
     parseWrapNativeTransferEvent,
     parseUnwrapNativeTransferEvent,
 } from './events/wrap_unwrap_native_events';
+export { parseERC20TransferEvent } from './events/erc20_transfer_events';

src/scripts/pull_and_save_block_events.ts

@@ -104,6 +104,12 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock): ParsedFullBlock {
 
 function parseTransactionEvents(transaction: FullTransaction): ParsedTransaction {
     const parsedTransaction = parseTransaction(transaction);
+    if (parsedTransaction.input === '0x') {
+        return {
+            parsedTransaction: null,
+            parsedEvents: null,
+        };
+    }
 
     const nestedParsedEvents: TypedEvents[] = eventScrperProps.map((props: EventScraperProps): TypedEvents => {
         if (props.enabled) {
@@ -134,13 +140,10 @@ function parseTransactionEvents(transaction: FullTransaction): ParsedTransaction
         };
     });
 
-    const foundScrapedEventInTx = nestedParsedEvents.map((npe) => npe.events).flat().length > 0;
-
     return {
         // We need all reverted txs for joining with traces later
         // toString is needed because we libraries do not match
-        parsedTransaction:
-            foundScrapedEventInTx || transaction.status!.toString() === 'false' ? parsedTransaction : null,
+        parsedTransaction: parsedTransaction,
         parsedEvents: nestedParsedEvents,
     };
 }