From 5fa45e9680ec7994dc9a41c3c3acc31089f6fcbd Mon Sep 17 00:00:00 2001 From: Yijing Date: Wed, 9 Oct 2024 15:38:56 -0500 Subject: [PATCH 1/9] test fix --- src/filters/erc20_transfer_events.ts | 34 ++++++++++++---- src/scripts/pull_and_save_block_events.ts | 48 +++++++++++++++++------ 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/src/filters/erc20_transfer_events.ts b/src/filters/erc20_transfer_events.ts index ed5e3c7a..6e1a0e18 100644 --- a/src/filters/erc20_transfer_events.ts +++ b/src/filters/erc20_transfer_events.ts @@ -2,23 +2,41 @@ import { Web3Source } from '../data_sources/events/web3'; import { Event, Transaction } from '../entities'; import { getParseTxsAsync } from '../scripts/utils/web3_utils'; -export async function filterERC20TransferEventsGetContext(events: Event[], web3Source: Web3Source): Promise { +export async function filterERC20TransferEventsGetContext( + events: Event[], + web3Source: Web3Source, + allowedTxnList?: Set, +): Promise { if (events.length > 0) { const txHashes = events.map((log: Event) => log.transactionHash); - const txData = await getParseTxsAsync(web3Source, txHashes); - const filteredTxsHashes = txData.parsedTxs - .filter((tx: Transaction) => tx.quoteId) - .map((tx: Transaction) => tx.transactionHash); + let validTxHashSet: Set; + if (allowedTxnList) { + validTxHashSet = allowedTxnList; + } else { + const txData = await getParseTxsAsync(web3Source, txHashes); + const filteredTxsHashes = txData.parsedTxs + .filter((tx: Transaction) => tx.quoteId) + .map((tx: Transaction) => tx.transactionHash); + + validTxHashSet = new Set(filteredTxsHashes); + } - const validTxHashSet = new Set(filteredTxsHashes); const filteredLogs = events.filter((log: Event) => validTxHashSet.has(log.transactionHash)); return filteredLogs.filter((e) => e !== null); } return []; } -export function filterERC20TransferEvents(events: Event[], transaction: Transaction): Event[] { - if (transaction.quoteId) { +export function filterERC20TransferEvents( + events: Event[], + transaction: Transaction, + allowedTxnList?: Set, +): Event[] { + if (allowedTxnList) { + if (allowedTxnList.has(transaction.transactionHash)) { + return events.filter((e) => e !== null); + } + } else if (transaction.quoteId) { return events.filter((e) => e !== null); } return []; diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index ea804c4d..1660f75c 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -73,15 +73,17 @@ const provider = web3Factory.getRpcProvider({ }); const web3Source = new Web3Source(provider, EVM_RPC_URL); -function parseBlockTransactionsEvents(fullBlock: FullBlock): ParsedFullBlock { +function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set): ParsedFullBlock { const parsedBlock = parseBlock({ ...fullBlock, transactions: [''] }); const usefullTxs: ParsedTransaction[] = fullBlock.transactions .map((transaction: FullTransaction): ParsedTransaction | null => { - const parsedTransactionEvents = parseTransactionEvents(transaction); - - if (parsedTransactionEvents.parsedTransaction !== null) { - return parsedTransactionEvents; + // Only parse transactions if they are in the allowed list or if no list is provided. + if (!allowedTxnList || allowedTxnList.has(transaction.hash)) { + const parsedTransactionEvents = parseTransactionEvents(transaction); + if (parsedTransactionEvents.parsedTransaction !== null) { + return parsedTransactionEvents; + } } return null; }) @@ -270,6 +272,7 @@ async function getParseSaveBlocksTransactionsEvents( producer: Producer | null, newBlocks: EVMBlock[], allowPartialSuccess: boolean, + allowedTxnList?: Set, ): Promise { const blockNumbers = newBlocks.map((newBlock) => newBlock.number!); @@ -280,7 +283,7 @@ async function getParseSaveBlocksTransactionsEvents( const newBlocksReceipts = await web3Source.getBatchBlockReceiptsAsync(blockNumbers); const filteredNewBlocksReceipts = newBlocksReceipts.filter( - (blockReciepts) => blockReciepts !== null && blockReciepts !== undefined, + (blockReceipts) => blockReceipts !== null && blockReceipts !== undefined, ); if (newBlocksReceipts.length !== filteredNewBlocksReceipts.length) { @@ -288,12 +291,12 @@ async function getParseSaveBlocksTransactionsEvents( return false; } const { nullOnlyAtEnd } = newBlocksReceipts.reduce( - (state, blockReciepts) => { - if (state.hasSeenNull && blockReciepts !== null) { + (state, blockReceipts) => { + if (state.hasSeenNull && blockReceipts !== null) { state.nullOnlyAtEnd = false; } - if (newBlocksReceipts === null) { + if (blockReceipts === null) { state.hasSeenNull = true; } return state; @@ -302,9 +305,9 @@ async function getParseSaveBlocksTransactionsEvents( ); if (nullOnlyAtEnd) { - logger.info('Last block(s) reciepts not found, retrying that block(s) on the next run'); + logger.info('Last block(s) receipts not found, retrying those block(s) on the next run'); } else { - logger.error("Missing intermideate block reciepts, can't continue. Retrying next run"); + logger.error("Missing intermediate block receipts, can't continue. Retrying next run"); logger.error(newBlocksReceipts); return false; } @@ -327,7 +330,9 @@ async function getParseSaveBlocksTransactionsEvents( return { ...newBlocks[blockIndex], transactions: transactionsWithLogs }; }); - const parsedFullBlocks = fullBlocks.map(parseBlockTransactionsEvents); + const parsedFullBlocks = fullBlocks.map((fullBlock) => + parseBlockTransactionsEvents(fullBlock, allowedTxnList) + ); const eventTables = eventScrperProps .filter((props) => props.enabled) @@ -371,9 +376,26 @@ export class BlockEventsScraper { const blockNumbers = oldestBlocksToBackfill.map( (backfillBlock: { block_number: number }) => backfillBlock.block_number, ); + const allowedTxnListQuery = await connection.query( + `SELECT DISTINCT txn_hash + FROM ${SCHEMA}.tx_backfill + WHERE block_number IN (${blockNumbers.join(',')}) AND done = false` + ); + + const allowedTxnList = new Set( + allowedTxnListQuery.map((row: { txn_hash: string }) => row.txn_hash) + ); const newBlocks = await web3Source.getBatchBlockInfoAsync(blockNumbers, true); - const success = await getParseSaveBlocksTransactionsEvents(connection, producer, newBlocks, false); + + const success = await getParseSaveBlocksTransactionsEvents( + connection, + producer, + newBlocks, + false, + allowedTxnList + ); + if (success) { const newBlockNumbers = newBlocks.map((block) => block.number); const queryRunner = connection.createQueryRunner(); From abbe1bdc987aac8bc5989e3cbb30245a87e7e057 Mon Sep 17 00:00:00 2001 From: Yijing Date: Wed, 9 Oct 2024 16:17:05 -0500 Subject: [PATCH 2/9] local dev --- docker-compose-dev.yml | 82 +++++++++++------------------------------- 1 file changed, 21 insertions(+), 61 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index db574b77..141d4bb2 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -2,8 +2,6 @@ version: '3' services: event-pipeline-ethereum: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -15,10 +13,6 @@ services: SCHEMA: 'events' # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' EP_DEPLOYMENT_BLOCK: 10247094 MAX_BLOCKS_TO_SEARCH: 1000 MAX_BLOCKS_TO_PULL: 1000 @@ -57,10 +51,10 @@ services: # ONCHAIN_GOVERNANCE_START_BLOCK: 16990159 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' + networks: + - 0x-data-migrations_default - event-pipeline-bsc: - depends_on: - - postgres + event-pipeline-bsc: build: context: . dockerfile: Dockerfile.dev @@ -71,10 +65,6 @@ services: POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_bsc' MAX_BLOCKS_REORG: 50 - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' EP_DEPLOYMENT_BLOCK: 5375047 MAX_BLOCKS_TO_SEARCH: 2000 MAX_BLOCKS_TO_PULL: 5000 @@ -87,10 +77,10 @@ services: # NFT_FEATURE_START_BLOCK: 15860129 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c' + networks: + - 0x-data-migrations_default event-pipeline-polygon: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -100,10 +90,6 @@ services: CHAIN_ID: '137' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_polygon' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' ENABLE_PROMETHEUS_METRICS: "true" EP_DEPLOYMENT_BLOCK: 14391480 MAX_BLOCKS_TO_SEARCH: 1000 @@ -121,10 +107,10 @@ services: # META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270' + networks: + - 0x-data-migrations_default event-pipeline-avalanche: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -134,10 +120,6 @@ services: CHAIN_ID: '43114' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_avalanche' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' ENABLE_PROMETHEUS_METRICS: "true" EP_DEPLOYMENT_BLOCK: 3601700 MAX_BLOCKS_TO_SEARCH: 5000 @@ -150,10 +132,10 @@ services: NFT_FEATURE_START_BLOCK: 11816835 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7' + networks: + - 0x-data-migrations_default event-pipeline-fantom: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -163,10 +145,6 @@ services: CHAIN_ID: '250' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_fantom' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' EP_ADDRESS: "0xDEF189DeAEF76E379df891899eb5A00a94cBC250" ENABLE_PROMETHEUS_METRICS: "true" EP_DEPLOYMENT_BLOCK: 18855765 @@ -180,10 +158,10 @@ services: NFT_FEATURE_START_BLOCK: 32817445 FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' + networks: + - 0x-data-migrations_default event-pipeline-celo: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -193,10 +171,6 @@ services: CHAIN_ID: '42220' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_celo' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' ENABLE_PROMETHEUS_METRICS: "true" EP_DEPLOYMENT_BLOCK: 9350111 MAX_BLOCKS_TO_SEARCH: 5000 @@ -204,10 +178,10 @@ services: MINUTES_BETWEEN_RUNS: 1 FEAT_NFT: "true" NFT_FEATURE_START_BLOCK: 11820000 + networks: + - 0x-data-migrations_default event-pipeline-optimism: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -217,10 +191,6 @@ services: CHAIN_ID: '10' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_optimism' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' EP_DEPLOYMENT_BLOCK: 1691335 MAX_BLOCKS_TO_SEARCH: 1000 MAX_BLOCKS_TO_PULL: 1000 @@ -230,10 +200,10 @@ services: NFT_FEATURE_START_BLOCK: 4214981 FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" + networks: + - 0x-data-migrations_default event-pipeline-arbitrum: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -243,10 +213,6 @@ services: CHAIN_ID: '42161' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events_arbitrum' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' EP_DEPLOYMENT_BLOCK: 4050733 MAX_BLOCKS_TO_SEARCH: 1000 MAX_BLOCKS_TO_PULL: 1000 @@ -256,10 +222,10 @@ services: NFT_FEATURE_START_BLOCK: 4050733 FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' + networks: + - 0x-data-migrations_default event-pipeline-base: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -275,12 +241,10 @@ services: SECONDS_BETWEEN_RUNS: 1 FEAT_NFT: "true" NFT_FEATURE_START_BLOCK: 1410394 - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4200000000000000000000000000000000000006' + networks: + - 0x-data-migrations_default event-pipeline-scroll: build: @@ -370,8 +334,6 @@ services: - 0x-data-migrations_default token-scraper-ethereum: - depends_on: - - postgres build: context: . dockerfile: Dockerfile.dev @@ -381,10 +343,6 @@ services: CHAIN_ID: '1' POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' SCHEMA: 'events' - KAFKA_BROKERS: '${KAFKA_BROKERS}' - KAFKA_SSL: '${KAFKA_SSL}' - KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' LOG_LEVEL: "info" FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 @@ -393,6 +351,8 @@ services: MAX_BLOCKS_TO_PULL: 1000 MAX_TX_TO_PULL: 1000 SECONDS_BETWEEN_RUNS: 5 + networks: + - 0x-data-migrations_default networks: 0x-data-migrations_default: From f0026f216819cc476dc49508a27cb2860a78cc7c Mon Sep 17 00:00:00 2001 From: Yijing Date: Wed, 9 Oct 2024 17:54:45 -0500 Subject: [PATCH 3/9] fix --- docker-compose-dev.yml | 714 ++++++++++++---------- src/events.ts | 4 +- src/filters/erc20_transfer_events.ts | 4 +- src/scripts/pull_and_save_block_events.ts | 15 +- 4 files changed, 397 insertions(+), 340 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 141d4bb2..d59ad346 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,229 +1,259 @@ version: '3' services: - event-pipeline-ethereum: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' - CHAIN_ID: '1' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events' - # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" - # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 - EP_DEPLOYMENT_BLOCK: 10247094 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 1000 - MAX_TX_TO_PULL: 1000 - BLOCK_FINALITY_THRESHOLD: 0 - SECONDS_BETWEEN_RUNS: 1 - # FEAT_TRANSFORMED_ERC20_EVENT: "true" - # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" - # UNISWAP_V2_VIP_SWAP_SOURCES: "UniswapV2,SushiSwap" - # UNISWAP_V2_VIP_SWAP_START_BLOCK: 10917104 - # FEAT_UNISWAP_V3_VIP_SWAP_EVENT: "true" - # UNISWAP_V3_VIP_SWAP_START_BLOCK: 12553659 - # FEAT_UNISWAP_V3_SWAP_EVENT: "true" - # UNISWAP_V3_SWAP_START_BLOCK: 16670838 - # FEAT_LIMIT_ORDERS: "true" - # V4_NATIVE_FILL_START_BLOCK: "11591021" - # FEAT_PLP_SWAP_EVENT: "true" - # PLP_VIP_START_BLOCK: 11377457 - # FEAT_OTC_ORDERS: "true" - # OTC_ORDERS_FEATURE_START_BLOCK: 13143075 - # FEAT_CANCEL_EVENTS: "true" - # FEAT_STAKING: "true" - # STAKING_DEPLOYMENT_BLOCK: 8952581 - # FEAT_RFQ_EVENT: "true" - # FEAT_V3_NATIVE_FILL: "true" - # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - # FLASHWALLET_ADDRESS: "0x22f9dcf4647084d6c31b2765f6910cd85c178c18" - # FLASHWALLET_DEPLOYMENT_BLOCK: 12231666 - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 14258205 - # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" - # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "UniswapV2:0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f:10000835,SushiSwap:0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac:10794229" - # FEAT_UNISWAP_V2_SYNC_EVENT: "true" - # UNISWAP_V2_SYNC_START_BLOCK: 10000835 - # FEAT_ONCHAIN_GOVERNANCE: "true" - # ONCHAIN_GOVERNANCE_START_BLOCK: 16990159 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' - networks: - - 0x-data-migrations_default + # event-pipeline-ethereum: + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' + # CHAIN_ID: '1' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events' + # # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" + # # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # EP_DEPLOYMENT_BLOCK: 10247094 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 1000 + # MAX_TX_TO_PULL: 1000 + # BLOCK_FINALITY_THRESHOLD: 0 + # SECONDS_BETWEEN_RUNS: 1 + # # FEAT_TRANSFORMED_ERC20_EVENT: "true" + # # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" + # # UNISWAP_V2_VIP_SWAP_SOURCES: "UniswapV2,SushiSwap" + # # UNISWAP_V2_VIP_SWAP_START_BLOCK: 10917104 + # # FEAT_UNISWAP_V3_VIP_SWAP_EVENT: "true" + # # UNISWAP_V3_VIP_SWAP_START_BLOCK: 12553659 + # # FEAT_UNISWAP_V3_SWAP_EVENT: "true" + # # UNISWAP_V3_SWAP_START_BLOCK: 16670838 + # # FEAT_LIMIT_ORDERS: "true" + # # V4_NATIVE_FILL_START_BLOCK: "11591021" + # # FEAT_PLP_SWAP_EVENT: "true" + # # PLP_VIP_START_BLOCK: 11377457 + # # FEAT_OTC_ORDERS: "true" + # # OTC_ORDERS_FEATURE_START_BLOCK: 13143075 + # # FEAT_CANCEL_EVENTS: "true" + # # FEAT_STAKING: "true" + # # STAKING_DEPLOYMENT_BLOCK: 8952581 + # # FEAT_RFQ_EVENT: "true" + # # FEAT_V3_NATIVE_FILL: "true" + # # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + # # FLASHWALLET_ADDRESS: "0x22f9dcf4647084d6c31b2765f6910cd85c178c18" + # # FLASHWALLET_DEPLOYMENT_BLOCK: 12231666 + # # FEAT_NFT: "true" + # # NFT_FEATURE_START_BLOCK: 14258205 + # # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" + # # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "UniswapV2:0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f:10000835,SushiSwap:0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac:10794229" + # # FEAT_UNISWAP_V2_SYNC_EVENT: "true" + # # UNISWAP_V2_SYNC_START_BLOCK: 10000835 + # # FEAT_ONCHAIN_GOVERNANCE: "true" + # # ONCHAIN_GOVERNANCE_START_BLOCK: 16990159 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' - event-pipeline-bsc: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_BSC}' - CHAIN_ID: '56' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_bsc' - MAX_BLOCKS_REORG: 50 - EP_DEPLOYMENT_BLOCK: 5375047 - MAX_BLOCKS_TO_SEARCH: 2000 - MAX_BLOCKS_TO_PULL: 5000 - SECONDS_BETWEEN_RUNS: 1 - # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" - # UNISWAP_V2_VIP_SWAP_SOURCES: "PancakeSwap,BakerySwap,SushiSwap,CafeSwap,SwapLiquidity,ApeSwapFinance,CheeseSwap,Swap" - # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" - # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "PancakeSwap:0xbcfccbde45ce874adcb698cc183debcf17952812:586851,PancakeSwap!v2:0xca143ce32fe78f1f7019d7d551a6402fc5350c73:6809737,BakerySwap:0x01bf7c66c6bd861915cdaae475042d3c4bae16a7:470617,SushiSwap:0xc35dadb65012ec5796536bd9864ed8773abc74c4:5205069,CafeSwap:0x3e708fdbe3ada63fc94f8f61811196f1302137ad:5865260,SwapLiquidity:0x553990f2cba90272390f62c5bdb1681ffc899675:784352,ApeSwapFinance:0x0841bd0b734e4f5853f0dd8d7ea041c241fb0da6:4855901,CheeseSwap:0xdd538e4fd1b69b7863e1f741213276a6cf1efb3b:1569172" - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 15860129 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c' - networks: - - 0x-data-migrations_default + # event-pipeline-bsc: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_BSC}' + # CHAIN_ID: '56' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_bsc' + # MAX_BLOCKS_REORG: 50 + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # EP_DEPLOYMENT_BLOCK: 5375047 + # MAX_BLOCKS_TO_SEARCH: 2000 + # MAX_BLOCKS_TO_PULL: 5000 + # SECONDS_BETWEEN_RUNS: 1 + # # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" + # # UNISWAP_V2_VIP_SWAP_SOURCES: "PancakeSwap,BakerySwap,SushiSwap,CafeSwap,SwapLiquidity,ApeSwapFinance,CheeseSwap,Swap" + # # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" + # # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "PancakeSwap:0xbcfccbde45ce874adcb698cc183debcf17952812:586851,PancakeSwap!v2:0xca143ce32fe78f1f7019d7d551a6402fc5350c73:6809737,BakerySwap:0x01bf7c66c6bd861915cdaae475042d3c4bae16a7:470617,SushiSwap:0xc35dadb65012ec5796536bd9864ed8773abc74c4:5205069,CafeSwap:0x3e708fdbe3ada63fc94f8f61811196f1302137ad:5865260,SwapLiquidity:0x553990f2cba90272390f62c5bdb1681ffc899675:784352,ApeSwapFinance:0x0841bd0b734e4f5853f0dd8d7ea041c241fb0da6:4855901,CheeseSwap:0xdd538e4fd1b69b7863e1f741213276a6cf1efb3b:1569172" + # # FEAT_NFT: "true" + # # NFT_FEATURE_START_BLOCK: 15860129 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c' - event-pipeline-polygon: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_POLYGON}' - CHAIN_ID: '137' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_polygon' - ENABLE_PROMETHEUS_METRICS: "true" - EP_DEPLOYMENT_BLOCK: 14391480 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 1000 - MINUTES_BETWEEN_RUNS: 1 - # FEAT_SLINGSHOT_TRADE_EVENT: "true" - # SLINGSHOT_DEPLOYMENT_BLOCK: 14500000 - # FEAT_LIMIT_ORDERS: "true" - # V4_NATIVE_FILL_START_BLOCK: "17587182" - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 25697854 - # FEAT_OTC_ORDERS: "true" - # OTC_ORDERS_FEATURE_START_BLOCK: 26479109 - # FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" - # META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270' - networks: - - 0x-data-migrations_default + # event-pipeline-polygon: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_POLYGON}' + # CHAIN_ID: '137' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_polygon' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # ENABLE_PROMETHEUS_METRICS: "true" + # EP_DEPLOYMENT_BLOCK: 14391480 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 1000 + # MINUTES_BETWEEN_RUNS: 1 + # # FEAT_SLINGSHOT_TRADE_EVENT: "true" + # # SLINGSHOT_DEPLOYMENT_BLOCK: 14500000 + # # FEAT_LIMIT_ORDERS: "true" + # # V4_NATIVE_FILL_START_BLOCK: "17587182" + # # FEAT_NFT: "true" + # # NFT_FEATURE_START_BLOCK: 25697854 + # # FEAT_OTC_ORDERS: "true" + # # OTC_ORDERS_FEATURE_START_BLOCK: 26479109 + # # FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" + # # META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270' - event-pipeline-avalanche: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_AVALANCHE}' - CHAIN_ID: '43114' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_avalanche' - ENABLE_PROMETHEUS_METRICS: "true" - EP_DEPLOYMENT_BLOCK: 3601700 - MAX_BLOCKS_TO_SEARCH: 5000 - MAX_BLOCKS_TO_PULL: 2000 - MINUTES_BETWEEN_RUNS: 1 - FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" - FLASHWALLET_DEPLOYMENT_BLOCK: 11805869 - FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: 11816835 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7' - networks: - - 0x-data-migrations_default + # event-pipeline-avalanche: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_AVALANCHE}' + # CHAIN_ID: '43114' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_avalanche' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # ENABLE_PROMETHEUS_METRICS: "true" + # EP_DEPLOYMENT_BLOCK: 3601700 + # MAX_BLOCKS_TO_SEARCH: 5000 + # MAX_BLOCKS_TO_PULL: 2000 + # MINUTES_BETWEEN_RUNS: 1 + # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + # FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" + # FLASHWALLET_DEPLOYMENT_BLOCK: 11805869 + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 11816835 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7' - event-pipeline-fantom: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_FANTOM}' - CHAIN_ID: '250' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_fantom' - EP_ADDRESS: "0xDEF189DeAEF76E379df891899eb5A00a94cBC250" - ENABLE_PROMETHEUS_METRICS: "true" - EP_DEPLOYMENT_BLOCK: 18855765 - MAX_BLOCKS_TO_SEARCH: 2000 - MAX_BLOCKS_TO_PULL: 1000 - SECONDS_BETWEEN_RUNS: 60 - FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - FLASHWALLET_ADDRESS: "0xb4d961671cadfed687e040b076eee29840c142e5" - FLASHWALLET_DEPLOYMENT_BLOCK: 18855797 - FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: 32817445 - FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' - networks: - - 0x-data-migrations_default + # event-pipeline-fantom: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_FANTOM}' + # CHAIN_ID: '250' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_fantom' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # EP_ADDRESS: "0xDEF189DeAEF76E379df891899eb5A00a94cBC250" + # ENABLE_PROMETHEUS_METRICS: "true" + # EP_DEPLOYMENT_BLOCK: 18855765 + # MAX_BLOCKS_TO_SEARCH: 2000 + # MAX_BLOCKS_TO_PULL: 1000 + # SECONDS_BETWEEN_RUNS: 60 + # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + # FLASHWALLET_ADDRESS: "0xb4d961671cadfed687e040b076eee29840c142e5" + # FLASHWALLET_DEPLOYMENT_BLOCK: 18855797 + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 32817445 + # FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' - event-pipeline-celo: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_CELO}' - CHAIN_ID: '42220' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_celo' - ENABLE_PROMETHEUS_METRICS: "true" - EP_DEPLOYMENT_BLOCK: 9350111 - MAX_BLOCKS_TO_SEARCH: 5000 - MAX_BLOCKS_TO_PULL: 2000 - MINUTES_BETWEEN_RUNS: 1 - FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: 11820000 - networks: - - 0x-data-migrations_default + # event-pipeline-celo: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_CELO}' + # CHAIN_ID: '42220' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_celo' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # ENABLE_PROMETHEUS_METRICS: "true" + # EP_DEPLOYMENT_BLOCK: 9350111 + # MAX_BLOCKS_TO_SEARCH: 5000 + # MAX_BLOCKS_TO_PULL: 2000 + # MINUTES_BETWEEN_RUNS: 1 + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 11820000 - event-pipeline-optimism: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_OPTIMISM}' - CHAIN_ID: '10' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_optimism' - EP_DEPLOYMENT_BLOCK: 1691335 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 1000 - SECONDS_BETWEEN_RUNS: 30 - EP_ADDRESS: "0xdef1abe32c034e558cdd535791643c58a13acc10" - FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: 4214981 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" - networks: - - 0x-data-migrations_default + # event-pipeline-optimism: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_OPTIMISM}' + # CHAIN_ID: '10' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_optimism' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # EP_DEPLOYMENT_BLOCK: 1691335 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 1000 + # SECONDS_BETWEEN_RUNS: 30 + # EP_ADDRESS: "0xdef1abe32c034e558cdd535791643c58a13acc10" + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 4214981 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" - event-pipeline-arbitrum: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_ARBITRUM}' - CHAIN_ID: '42161' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_arbitrum' - EP_DEPLOYMENT_BLOCK: 4050733 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 1000 - SECONDS_BETWEEN_RUNS: 30 - EP_ADDRESS: "0xdef1c0ded9bec7f1a1670819833240f027b25eff" - FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: 4050733 - FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' - networks: - - 0x-data-migrations_default + # event-pipeline-arbitrum: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_ARBITRUM}' + # CHAIN_ID: '42161' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events_arbitrum' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # EP_DEPLOYMENT_BLOCK: 4050733 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 1000 + # SECONDS_BETWEEN_RUNS: 30 + # EP_ADDRESS: "0xdef1c0ded9bec7f1a1670819833240f027b25eff" + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 4050733 + # FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' event-pipeline-base: build: @@ -231,128 +261,156 @@ services: dockerfile: Dockerfile.dev restart: always environment: - ETHEREUM_RPC_URL: '${RPC_URL_BASE}' - CHAIN_ID: '8453' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events_base' - EP_DEPLOYMENT_BLOCK: 1410394 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 100 - SECONDS_BETWEEN_RUNS: 1 + EVM_RPC_URL: "${RPC_URL_BASE}" # Use a secret or .env file for sensitive data + CHAIN_ID: "8453" + SCHEMA: "events_base" + LOG_LEVEL: "info" + SCRAPER_MODE: "BLOCKS" + BLOCK_RECEIPTS_MODE: "alchemy" + ENABLE_PROMETHEUS_METRICS: "true" + SECONDS_BETWEEN_RUNS: "1" + RESCRAPE_BLOCKS: "10" + FEAT_ZEROEX_EXCHANGE_PROXY: "true" + EP_DEPLOYMENT_BLOCK: "1410394" + SETTLER_DEPLOYMENT_BLOCK: "12723120" + MAX_BLOCKS_TO_SEARCH: "500" + MAX_BLOCKS_TO_PULL: "102" + MAX_BLOCKS_REORG: "300" + FEAT_TRANSFORMED_ERC20_EVENT: "true" + FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" + FLASHWALLET_DEPLOYMENT_BLOCK: "1410394" FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: 1410394 + NFT_FEATURE_START_BLOCK: "1410394" + FEAT_OTC_ORDERS: "true" + OTC_ORDERS_FEATURE_START_BLOCK: "1410394" + BLOCK_FINALITY_THRESHOLD: "0" # No longer used FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4200000000000000000000000000000000000006' - networks: - - 0x-data-migrations_default - - event-pipeline-scroll: - build: - context: . - dockerfile: Dockerfile.dev - platform: linux/amd64 - restart: always - environment: - SCRAPER_MODE: 'BLOCKS' - EVM_RPC_URL: '${RPC_URL_SCROLL}' - CHAIN_ID: '534352' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' - SCHEMA: 'events_scroll' - FEAT_ZEROEX_EXCHANGE_PROXY: "false" - SETTLER_DEPLOYMENT_BLOCK: 7474150 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 100 - SECONDS_BETWEEN_RUNS: 1 - RESCRAPE_BLOCKS: 10 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x5300000000000000000000000000000000000004' + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" + FEAT_SOCKET_BRIDGE_EVENT: "true" + SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" + SOCKET_BRIDGE_EVENT_START_BLOCK: "6003127" FEAT_TOKENS_FROM_TRANSFERS: "true" + FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" + META_TRANSACTION_EXECUTED_START_BLOCK: "10540349" FEAT_ERC20_TRANSFER_ALL: "true" FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - FEAT_SETTLER_RFQ_ORDER_EVENT: "true" + FEAT_SETTLER_RFQ_ORDER_EVENT: "false" + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' networks: - 0x-data-migrations_default - event-pipeline-blast: - build: - context: . - dockerfile: Dockerfile.dev - platform: linux/amd64 - restart: always - environment: - SCRAPER_MODE: 'EVENTS' - EVM_RPC_URL: '${RPC_URL_BLAST}' - CHAIN_ID: '81457' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' - SCHEMA: 'events_blast' - FEAT_ZEROEX_EXCHANGE_PROXY: "false" - SETTLER_DEPLOYMENT_BLOCK: 6124420 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 100 - SECONDS_BETWEEN_RUNS: 1 - RESCRAPE_BLOCKS: 10 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4300000000000000000000000000000000000004' - FEAT_TOKENS_FROM_TRANSFERS: "true" - FEAT_ERC20_TRANSFER_ALL: "true" - FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - TOKENS_FROM_TRANSFERS_START_BLOCK: "1" - SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" - SOCKET_BRIDGE_EVENT_START_BLOCK: "6124420" - networks: - - 0x-data-migrations_default + # event-pipeline-scroll: + # build: + # context: . + # dockerfile: Dockerfile.dev + # platform: linux/amd64 + # restart: always + # environment: + # SCRAPER_MODE: 'BLOCKS' + # EVM_RPC_URL: '${RPC_URL_SCROLL}' + # CHAIN_ID: '534352' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' + # SCHEMA: 'events_scroll' + # FEAT_ZEROEX_EXCHANGE_PROXY: "false" + # SETTLER_DEPLOYMENT_BLOCK: 7474150 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 100 + # SECONDS_BETWEEN_RUNS: 1 + # RESCRAPE_BLOCKS: 10 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x5300000000000000000000000000000000000004' + # FEAT_TOKENS_FROM_TRANSFERS: "true" + # FEAT_ERC20_TRANSFER_ALL: "true" + # FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" + # FEAT_SETTLER_RFQ_ORDER_EVENT: "true" + # networks: + # - 0x-data-migrations_default - event-pipeline-linea: - build: - context: . - dockerfile: Dockerfile.dev - platform: linux/amd64 - restart: always - environment: - SCRAPER_MODE: 'EVENTS' - EVM_RPC_URL: '${RPC_URL_LINEA}' - CHAIN_ID: '59144' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' - SCHEMA: 'events_linea' - FEAT_ZEROEX_EXCHANGE_PROXY: "false" - SETTLER_DEPLOYMENT_BLOCK: 6917652 - MAX_BLOCKS_TO_SEARCH: 125 - MAX_BLOCKS_TO_PULL: 25 - MAX_BLOCKS_REORG: 125 - SECONDS_BETWEEN_RUNS: 1 - RESCRAPE_BLOCKS: 10 - FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xe5d7c2a44ffddf6b295a15c148167daaaf5cf34f' - FEAT_TOKENS_FROM_TRANSFERS: "true" - FEAT_ERC20_TRANSFER_ALL: "true" - FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - TOKENS_FROM_TRANSFERS_START_BLOCK: "1" - FEAT_SOCKET_BRIDGE_EVENT: "true" - SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" - SOCKET_BRIDGE_EVENT_START_BLOCK: "6917652" - networks: - - 0x-data-migrations_default + # event-pipeline-blast: + # build: + # context: . + # dockerfile: Dockerfile.dev + # platform: linux/amd64 + # restart: always + # environment: + # SCRAPER_MODE: 'EVENTS' + # EVM_RPC_URL: '${RPC_URL_BLAST}' + # CHAIN_ID: '81457' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' + # SCHEMA: 'events_blast' + # FEAT_ZEROEX_EXCHANGE_PROXY: "false" + # SETTLER_DEPLOYMENT_BLOCK: 6124420 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 100 + # SECONDS_BETWEEN_RUNS: 1 + # RESCRAPE_BLOCKS: 10 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4300000000000000000000000000000000000004' + # FEAT_TOKENS_FROM_TRANSFERS: "true" + # FEAT_ERC20_TRANSFER_ALL: "true" + # FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" + # TOKENS_FROM_TRANSFERS_START_BLOCK: "1" + # SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" + # SOCKET_BRIDGE_EVENT_START_BLOCK: "6124420" + # networks: + # - 0x-data-migrations_default - token-scraper-ethereum: - build: - context: . - dockerfile: Dockerfile.dev - restart: always - environment: - ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' - CHAIN_ID: '1' - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - SCHEMA: 'events' - LOG_LEVEL: "info" - FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" - TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 - EP_DEPLOYMENT_BLOCK: 10247094 - MAX_BLOCKS_TO_SEARCH: 1000 - MAX_BLOCKS_TO_PULL: 1000 - MAX_TX_TO_PULL: 1000 - SECONDS_BETWEEN_RUNS: 5 - networks: - - 0x-data-migrations_default + # event-pipeline-linea: + # build: + # context: . + # dockerfile: Dockerfile.dev + # platform: linux/amd64 + # restart: always + # environment: + # SCRAPER_MODE: 'EVENTS' + # EVM_RPC_URL: '${RPC_URL_LINEA}' + # CHAIN_ID: '59144' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' + # SCHEMA: 'events_linea' + # FEAT_ZEROEX_EXCHANGE_PROXY: "false" + # SETTLER_DEPLOYMENT_BLOCK: 6917652 + # MAX_BLOCKS_TO_SEARCH: 125 + # MAX_BLOCKS_TO_PULL: 25 + # MAX_BLOCKS_REORG: 125 + # SECONDS_BETWEEN_RUNS: 1 + # RESCRAPE_BLOCKS: 10 + # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xe5d7c2a44ffddf6b295a15c148167daaaf5cf34f' + # FEAT_TOKENS_FROM_TRANSFERS: "true" + # FEAT_ERC20_TRANSFER_ALL: "true" + # FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" + # TOKENS_FROM_TRANSFERS_START_BLOCK: "1" + # FEAT_SOCKET_BRIDGE_EVENT: "true" + # SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" + # SOCKET_BRIDGE_EVENT_START_BLOCK: "6917652" + # networks: + # - 0x-data-migrations_default + + # token-scraper-ethereum: + # depends_on: + # - postgres + # build: + # context: . + # dockerfile: Dockerfile.dev + # restart: always + # environment: + # ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' + # CHAIN_ID: '1' + # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + # SCHEMA: 'events' + # KAFKA_BROKERS: '${KAFKA_BROKERS}' + # KAFKA_SSL: '${KAFKA_SSL}' + # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + # LOG_LEVEL: "info" + # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" + # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 + # EP_DEPLOYMENT_BLOCK: 10247094 + # MAX_BLOCKS_TO_SEARCH: 1000 + # MAX_BLOCKS_TO_PULL: 1000 + # MAX_TX_TO_PULL: 1000 + # SECONDS_BETWEEN_RUNS: 5 networks: 0x-data-migrations_default: diff --git a/src/events.ts b/src/events.ts index 4754a216..6675b4ff 100644 --- a/src/events.ts +++ b/src/events.ts @@ -265,8 +265,8 @@ export type EventScraperProps = { deleteOptions?: DeleteOptions; tokenMetadataMap?: TokenMetadataMap; postProcess?: any; - filterFunction?: (events: Event[], transaction: Transaction) => Event[]; - filterFunctionGetContext?: (events: Event[], web3Source: Web3Source) => Promise; + filterFunction?: (events: Event[], transaction: Transaction, allowedTxnList?: Set) => Event[]; + filterFunctionGetContext?: (events: Event[], web3Source: Web3Source, allowedTxnList?: Set) => Promise; }; export const eventScrperProps: EventScraperProps[] = [ diff --git a/src/filters/erc20_transfer_events.ts b/src/filters/erc20_transfer_events.ts index 6e1a0e18..3cf48fcf 100644 --- a/src/filters/erc20_transfer_events.ts +++ b/src/filters/erc20_transfer_events.ts @@ -10,7 +10,7 @@ export async function filterERC20TransferEventsGetContext( if (events.length > 0) { const txHashes = events.map((log: Event) => log.transactionHash); let validTxHashSet: Set; - if (allowedTxnList) { + if (allowedTxnList && allowedTxnList.size > 0) { validTxHashSet = allowedTxnList; } else { const txData = await getParseTxsAsync(web3Source, txHashes); @@ -32,7 +32,7 @@ export function filterERC20TransferEvents( transaction: Transaction, allowedTxnList?: Set, ): Event[] { - if (allowedTxnList) { + if (allowedTxnList && allowedTxnList.size > 0) { if (allowedTxnList.has(transaction.transactionHash)) { return events.filter((e) => e !== null); } diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index 1660f75c..8bc12ef2 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -78,9 +78,8 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set const usefullTxs: ParsedTransaction[] = fullBlock.transactions .map((transaction: FullTransaction): ParsedTransaction | null => { - // Only parse transactions if they are in the allowed list or if no list is provided. if (!allowedTxnList || allowedTxnList.has(transaction.hash)) { - const parsedTransactionEvents = parseTransactionEvents(transaction); + const parsedTransactionEvents = parseTransactionEvents(transaction, allowedTxnList); if (parsedTransactionEvents.parsedTransaction !== null) { return parsedTransactionEvents; } @@ -88,7 +87,7 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set return null; }) .filter((tx) => tx !== null) as ParsedTransaction[]; - + const parsedTransactions: Transaction[] = usefullTxs.map((tx) => tx!.parsedTransaction!); const parsedTransactionHashes: string[] = parsedTransactions.map((tx) => tx.transactionHash); @@ -104,7 +103,7 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set }; } -function parseTransactionEvents(transaction: FullTransaction): ParsedTransaction { +function parseTransactionEvents(transaction: FullTransaction, allowedTxnList?: Set): ParsedTransaction { if (transaction.input === '0x') { return { parsedTransaction: null, @@ -124,7 +123,7 @@ function parseTransactionEvents(transaction: FullTransaction): ParsedTransaction const parsedLogs = baseFilteredLogs.map((log: LogEntry) => props.parser(log)); const filteredLogs = props.filterFunction - ? props.filterFunction(parsedLogs, parsedTransaction) + ? props.filterFunction(parsedLogs, parsedTransaction, allowedTxnList) : parsedLogs; const postProcessedLogs = props.postProcess ? props.postProcess(filteredLogs) : filteredLogs; @@ -377,13 +376,13 @@ export class BlockEventsScraper { (backfillBlock: { block_number: number }) => backfillBlock.block_number, ); const allowedTxnListQuery = await connection.query( - `SELECT DISTINCT txn_hash + `SELECT DISTINCT transaction_hash FROM ${SCHEMA}.tx_backfill WHERE block_number IN (${blockNumbers.join(',')}) AND done = false` ); - const allowedTxnList = new Set( - allowedTxnListQuery.map((row: { txn_hash: string }) => row.txn_hash) + const allowedTxnList = new Set( + allowedTxnListQuery.map((row: { transaction_hash: string }) => row.transaction_hash) ); const newBlocks = await web3Source.getBatchBlockInfoAsync(blockNumbers, true); From cfc383b40aa9d78950592f34d4f6bc0521cd1a28 Mon Sep 17 00:00:00 2001 From: Yijing Date: Wed, 9 Oct 2024 18:02:00 -0500 Subject: [PATCH 4/9] make prettier --- src/events.ts | 6 +++++- src/scripts/pull_and_save_block_events.ts | 12 +++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/events.ts b/src/events.ts index 6675b4ff..82089e2e 100644 --- a/src/events.ts +++ b/src/events.ts @@ -266,7 +266,11 @@ export type EventScraperProps = { tokenMetadataMap?: TokenMetadataMap; postProcess?: any; filterFunction?: (events: Event[], transaction: Transaction, allowedTxnList?: Set) => Event[]; - filterFunctionGetContext?: (events: Event[], web3Source: Web3Source, allowedTxnList?: Set) => Promise; + filterFunctionGetContext?: ( + events: Event[], + web3Source: Web3Source, + allowedTxnList?: Set, + ) => Promise; }; export const eventScrperProps: EventScraperProps[] = [ diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index 8bc12ef2..bb8dec86 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -87,7 +87,7 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set return null; }) .filter((tx) => tx !== null) as ParsedTransaction[]; - + const parsedTransactions: Transaction[] = usefullTxs.map((tx) => tx!.parsedTransaction!); const parsedTransactionHashes: string[] = parsedTransactions.map((tx) => tx.transactionHash); @@ -329,9 +329,7 @@ async function getParseSaveBlocksTransactionsEvents( return { ...newBlocks[blockIndex], transactions: transactionsWithLogs }; }); - const parsedFullBlocks = fullBlocks.map((fullBlock) => - parseBlockTransactionsEvents(fullBlock, allowedTxnList) - ); + const parsedFullBlocks = fullBlocks.map((fullBlock) => parseBlockTransactionsEvents(fullBlock, allowedTxnList)); const eventTables = eventScrperProps .filter((props) => props.enabled) @@ -378,11 +376,11 @@ export class BlockEventsScraper { const allowedTxnListQuery = await connection.query( `SELECT DISTINCT transaction_hash FROM ${SCHEMA}.tx_backfill - WHERE block_number IN (${blockNumbers.join(',')}) AND done = false` + WHERE block_number IN (${blockNumbers.join(',')}) AND done = false`, ); const allowedTxnList = new Set( - allowedTxnListQuery.map((row: { transaction_hash: string }) => row.transaction_hash) + allowedTxnListQuery.map((row: { transaction_hash: string }) => row.transaction_hash), ); const newBlocks = await web3Source.getBatchBlockInfoAsync(blockNumbers, true); @@ -392,7 +390,7 @@ export class BlockEventsScraper { producer, newBlocks, false, - allowedTxnList + allowedTxnList, ); if (success) { From 497076c8afaee7c84a9255c442b077b37e8831c7 Mon Sep 17 00:00:00 2001 From: Yijing Date: Thu, 10 Oct 2024 12:08:06 -0500 Subject: [PATCH 5/9] fix --- src/filters/erc20_transfer_events.ts | 17 ++++++++++------- src/scripts/pull_and_save_block_events.ts | 8 +++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/filters/erc20_transfer_events.ts b/src/filters/erc20_transfer_events.ts index 3cf48fcf..5f02e39a 100644 --- a/src/filters/erc20_transfer_events.ts +++ b/src/filters/erc20_transfer_events.ts @@ -32,12 +32,15 @@ export function filterERC20TransferEvents( transaction: Transaction, allowedTxnList?: Set, ): Event[] { - if (allowedTxnList && allowedTxnList.size > 0) { - if (allowedTxnList.has(transaction.transactionHash)) { - return events.filter((e) => e !== null); - } - } else if (transaction.quoteId) { - return events.filter((e) => e !== null); + const filteredEvents = new Set(); + + if (allowedTxnList && allowedTxnList.size > 0 && allowedTxnList.has(transaction.transactionHash)) { + events.filter((e) => e !== null).forEach((e) => filteredEvents.add(e)); } - return []; + + if (transaction.quoteId) { + events.filter((e) => e !== null).forEach((e) => filteredEvents.add(e)); + } + + return Array.from(filteredEvents); } diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index bb8dec86..d82cb95d 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -78,11 +78,9 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set const usefullTxs: ParsedTransaction[] = fullBlock.transactions .map((transaction: FullTransaction): ParsedTransaction | null => { - if (!allowedTxnList || allowedTxnList.has(transaction.hash)) { - const parsedTransactionEvents = parseTransactionEvents(transaction, allowedTxnList); - if (parsedTransactionEvents.parsedTransaction !== null) { - return parsedTransactionEvents; - } + const parsedTransactionEvents = parseTransactionEvents(transaction, allowedTxnList); + if (parsedTransactionEvents.parsedTransaction !== null) { + return parsedTransactionEvents; } return null; }) From 425b3a87b4c8f6facefb07c29f9f9693360dc3b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Elizondo?= Date: Fri, 8 Nov 2024 06:10:21 -0600 Subject: [PATCH 6/9] Restores docker-compose-dev.yaml --- docker-compose-dev.yml | 754 ++++++++++++++++++++--------------------- 1 file changed, 368 insertions(+), 386 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index d59ad346..db574b77 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,416 +1,398 @@ version: '3' services: - # event-pipeline-ethereum: - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' - # CHAIN_ID: '1' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events' - # # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" - # # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # EP_DEPLOYMENT_BLOCK: 10247094 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 1000 - # MAX_TX_TO_PULL: 1000 - # BLOCK_FINALITY_THRESHOLD: 0 - # SECONDS_BETWEEN_RUNS: 1 - # # FEAT_TRANSFORMED_ERC20_EVENT: "true" - # # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" - # # UNISWAP_V2_VIP_SWAP_SOURCES: "UniswapV2,SushiSwap" - # # UNISWAP_V2_VIP_SWAP_START_BLOCK: 10917104 - # # FEAT_UNISWAP_V3_VIP_SWAP_EVENT: "true" - # # UNISWAP_V3_VIP_SWAP_START_BLOCK: 12553659 - # # FEAT_UNISWAP_V3_SWAP_EVENT: "true" - # # UNISWAP_V3_SWAP_START_BLOCK: 16670838 - # # FEAT_LIMIT_ORDERS: "true" - # # V4_NATIVE_FILL_START_BLOCK: "11591021" - # # FEAT_PLP_SWAP_EVENT: "true" - # # PLP_VIP_START_BLOCK: 11377457 - # # FEAT_OTC_ORDERS: "true" - # # OTC_ORDERS_FEATURE_START_BLOCK: 13143075 - # # FEAT_CANCEL_EVENTS: "true" - # # FEAT_STAKING: "true" - # # STAKING_DEPLOYMENT_BLOCK: 8952581 - # # FEAT_RFQ_EVENT: "true" - # # FEAT_V3_NATIVE_FILL: "true" - # # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - # # FLASHWALLET_ADDRESS: "0x22f9dcf4647084d6c31b2765f6910cd85c178c18" - # # FLASHWALLET_DEPLOYMENT_BLOCK: 12231666 - # # FEAT_NFT: "true" - # # NFT_FEATURE_START_BLOCK: 14258205 - # # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" - # # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "UniswapV2:0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f:10000835,SushiSwap:0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac:10794229" - # # FEAT_UNISWAP_V2_SYNC_EVENT: "true" - # # UNISWAP_V2_SYNC_START_BLOCK: 10000835 - # # FEAT_ONCHAIN_GOVERNANCE: "true" - # # ONCHAIN_GOVERNANCE_START_BLOCK: 16990159 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' + event-pipeline-ethereum: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' + CHAIN_ID: '1' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events' + # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" + # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 10247094 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + MAX_TX_TO_PULL: 1000 + BLOCK_FINALITY_THRESHOLD: 0 + SECONDS_BETWEEN_RUNS: 1 + # FEAT_TRANSFORMED_ERC20_EVENT: "true" + # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" + # UNISWAP_V2_VIP_SWAP_SOURCES: "UniswapV2,SushiSwap" + # UNISWAP_V2_VIP_SWAP_START_BLOCK: 10917104 + # FEAT_UNISWAP_V3_VIP_SWAP_EVENT: "true" + # UNISWAP_V3_VIP_SWAP_START_BLOCK: 12553659 + # FEAT_UNISWAP_V3_SWAP_EVENT: "true" + # UNISWAP_V3_SWAP_START_BLOCK: 16670838 + # FEAT_LIMIT_ORDERS: "true" + # V4_NATIVE_FILL_START_BLOCK: "11591021" + # FEAT_PLP_SWAP_EVENT: "true" + # PLP_VIP_START_BLOCK: 11377457 + # FEAT_OTC_ORDERS: "true" + # OTC_ORDERS_FEATURE_START_BLOCK: 13143075 + # FEAT_CANCEL_EVENTS: "true" + # FEAT_STAKING: "true" + # STAKING_DEPLOYMENT_BLOCK: 8952581 + # FEAT_RFQ_EVENT: "true" + # FEAT_V3_NATIVE_FILL: "true" + # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + # FLASHWALLET_ADDRESS: "0x22f9dcf4647084d6c31b2765f6910cd85c178c18" + # FLASHWALLET_DEPLOYMENT_BLOCK: 12231666 + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 14258205 + # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" + # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "UniswapV2:0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f:10000835,SushiSwap:0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac:10794229" + # FEAT_UNISWAP_V2_SYNC_EVENT: "true" + # UNISWAP_V2_SYNC_START_BLOCK: 10000835 + # FEAT_ONCHAIN_GOVERNANCE: "true" + # ONCHAIN_GOVERNANCE_START_BLOCK: 16990159 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' - # event-pipeline-bsc: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_BSC}' - # CHAIN_ID: '56' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_bsc' - # MAX_BLOCKS_REORG: 50 - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # EP_DEPLOYMENT_BLOCK: 5375047 - # MAX_BLOCKS_TO_SEARCH: 2000 - # MAX_BLOCKS_TO_PULL: 5000 - # SECONDS_BETWEEN_RUNS: 1 - # # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" - # # UNISWAP_V2_VIP_SWAP_SOURCES: "PancakeSwap,BakerySwap,SushiSwap,CafeSwap,SwapLiquidity,ApeSwapFinance,CheeseSwap,Swap" - # # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" - # # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "PancakeSwap:0xbcfccbde45ce874adcb698cc183debcf17952812:586851,PancakeSwap!v2:0xca143ce32fe78f1f7019d7d551a6402fc5350c73:6809737,BakerySwap:0x01bf7c66c6bd861915cdaae475042d3c4bae16a7:470617,SushiSwap:0xc35dadb65012ec5796536bd9864ed8773abc74c4:5205069,CafeSwap:0x3e708fdbe3ada63fc94f8f61811196f1302137ad:5865260,SwapLiquidity:0x553990f2cba90272390f62c5bdb1681ffc899675:784352,ApeSwapFinance:0x0841bd0b734e4f5853f0dd8d7ea041c241fb0da6:4855901,CheeseSwap:0xdd538e4fd1b69b7863e1f741213276a6cf1efb3b:1569172" - # # FEAT_NFT: "true" - # # NFT_FEATURE_START_BLOCK: 15860129 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c' + event-pipeline-bsc: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_BSC}' + CHAIN_ID: '56' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_bsc' + MAX_BLOCKS_REORG: 50 + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 5375047 + MAX_BLOCKS_TO_SEARCH: 2000 + MAX_BLOCKS_TO_PULL: 5000 + SECONDS_BETWEEN_RUNS: 1 + # FEAT_UNISWAP_V2_VIP_SWAP_EVENT: "true" + # UNISWAP_V2_VIP_SWAP_SOURCES: "PancakeSwap,BakerySwap,SushiSwap,CafeSwap,SwapLiquidity,ApeSwapFinance,CheeseSwap,Swap" + # FEAT_UNISWAP_V2_PAIR_CREATED_EVENT: "true" + # UNISWAP_V2_PAIR_CREATED_PROTOCOL_CONTRACT_ADDRESSES_AND_START_BLOCKS: "PancakeSwap:0xbcfccbde45ce874adcb698cc183debcf17952812:586851,PancakeSwap!v2:0xca143ce32fe78f1f7019d7d551a6402fc5350c73:6809737,BakerySwap:0x01bf7c66c6bd861915cdaae475042d3c4bae16a7:470617,SushiSwap:0xc35dadb65012ec5796536bd9864ed8773abc74c4:5205069,CafeSwap:0x3e708fdbe3ada63fc94f8f61811196f1302137ad:5865260,SwapLiquidity:0x553990f2cba90272390f62c5bdb1681ffc899675:784352,ApeSwapFinance:0x0841bd0b734e4f5853f0dd8d7ea041c241fb0da6:4855901,CheeseSwap:0xdd538e4fd1b69b7863e1f741213276a6cf1efb3b:1569172" + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 15860129 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c' - # event-pipeline-polygon: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_POLYGON}' - # CHAIN_ID: '137' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_polygon' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # ENABLE_PROMETHEUS_METRICS: "true" - # EP_DEPLOYMENT_BLOCK: 14391480 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 1000 - # MINUTES_BETWEEN_RUNS: 1 - # # FEAT_SLINGSHOT_TRADE_EVENT: "true" - # # SLINGSHOT_DEPLOYMENT_BLOCK: 14500000 - # # FEAT_LIMIT_ORDERS: "true" - # # V4_NATIVE_FILL_START_BLOCK: "17587182" - # # FEAT_NFT: "true" - # # NFT_FEATURE_START_BLOCK: 25697854 - # # FEAT_OTC_ORDERS: "true" - # # OTC_ORDERS_FEATURE_START_BLOCK: 26479109 - # # FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" - # # META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270' + event-pipeline-polygon: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_POLYGON}' + CHAIN_ID: '137' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_polygon' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 14391480 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + MINUTES_BETWEEN_RUNS: 1 + # FEAT_SLINGSHOT_TRADE_EVENT: "true" + # SLINGSHOT_DEPLOYMENT_BLOCK: 14500000 + # FEAT_LIMIT_ORDERS: "true" + # V4_NATIVE_FILL_START_BLOCK: "17587182" + # FEAT_NFT: "true" + # NFT_FEATURE_START_BLOCK: 25697854 + # FEAT_OTC_ORDERS: "true" + # OTC_ORDERS_FEATURE_START_BLOCK: 26479109 + # FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" + # META_TRANSACTION_EXECUTED_START_BLOCK: 39406300 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270' - # event-pipeline-avalanche: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_AVALANCHE}' - # CHAIN_ID: '43114' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_avalanche' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # ENABLE_PROMETHEUS_METRICS: "true" - # EP_DEPLOYMENT_BLOCK: 3601700 - # MAX_BLOCKS_TO_SEARCH: 5000 - # MAX_BLOCKS_TO_PULL: 2000 - # MINUTES_BETWEEN_RUNS: 1 - # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - # FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" - # FLASHWALLET_DEPLOYMENT_BLOCK: 11805869 - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 11816835 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7' + event-pipeline-avalanche: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_AVALANCHE}' + CHAIN_ID: '43114' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_avalanche' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 3601700 + MAX_BLOCKS_TO_SEARCH: 5000 + MAX_BLOCKS_TO_PULL: 2000 + MINUTES_BETWEEN_RUNS: 1 + FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" + FLASHWALLET_DEPLOYMENT_BLOCK: 11805869 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 11816835 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7' - # event-pipeline-fantom: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_FANTOM}' - # CHAIN_ID: '250' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_fantom' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # EP_ADDRESS: "0xDEF189DeAEF76E379df891899eb5A00a94cBC250" - # ENABLE_PROMETHEUS_METRICS: "true" - # EP_DEPLOYMENT_BLOCK: 18855765 - # MAX_BLOCKS_TO_SEARCH: 2000 - # MAX_BLOCKS_TO_PULL: 1000 - # SECONDS_BETWEEN_RUNS: 60 - # FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - # FLASHWALLET_ADDRESS: "0xb4d961671cadfed687e040b076eee29840c142e5" - # FLASHWALLET_DEPLOYMENT_BLOCK: 18855797 - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 32817445 - # FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' + event-pipeline-fantom: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_FANTOM}' + CHAIN_ID: '250' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_fantom' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_ADDRESS: "0xDEF189DeAEF76E379df891899eb5A00a94cBC250" + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 18855765 + MAX_BLOCKS_TO_SEARCH: 2000 + MAX_BLOCKS_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 60 + FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" + FLASHWALLET_ADDRESS: "0xb4d961671cadfed687e040b076eee29840c142e5" + FLASHWALLET_DEPLOYMENT_BLOCK: 18855797 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 32817445 + FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' - # event-pipeline-celo: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_CELO}' - # CHAIN_ID: '42220' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_celo' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # ENABLE_PROMETHEUS_METRICS: "true" - # EP_DEPLOYMENT_BLOCK: 9350111 - # MAX_BLOCKS_TO_SEARCH: 5000 - # MAX_BLOCKS_TO_PULL: 2000 - # MINUTES_BETWEEN_RUNS: 1 - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 11820000 + event-pipeline-celo: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_CELO}' + CHAIN_ID: '42220' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_celo' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + ENABLE_PROMETHEUS_METRICS: "true" + EP_DEPLOYMENT_BLOCK: 9350111 + MAX_BLOCKS_TO_SEARCH: 5000 + MAX_BLOCKS_TO_PULL: 2000 + MINUTES_BETWEEN_RUNS: 1 + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 11820000 - # event-pipeline-optimism: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_OPTIMISM}' - # CHAIN_ID: '10' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_optimism' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # EP_DEPLOYMENT_BLOCK: 1691335 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 1000 - # SECONDS_BETWEEN_RUNS: 30 - # EP_ADDRESS: "0xdef1abe32c034e558cdd535791643c58a13acc10" - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 4214981 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" + event-pipeline-optimism: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_OPTIMISM}' + CHAIN_ID: '10' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_optimism' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 1691335 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 30 + EP_ADDRESS: "0xdef1abe32c034e558cdd535791643c58a13acc10" + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 4214981 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" - # event-pipeline-arbitrum: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_ARBITRUM}' - # CHAIN_ID: '42161' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events_arbitrum' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # EP_DEPLOYMENT_BLOCK: 4050733 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 1000 - # SECONDS_BETWEEN_RUNS: 30 - # EP_ADDRESS: "0xdef1c0ded9bec7f1a1670819833240f027b25eff" - # FEAT_NFT: "true" - # NFT_FEATURE_START_BLOCK: 4050733 - # FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' + event-pipeline-arbitrum: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_ARBITRUM}' + CHAIN_ID: '42161' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_arbitrum' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + EP_DEPLOYMENT_BLOCK: 4050733 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 30 + EP_ADDRESS: "0xdef1c0ded9bec7f1a1670819833240f027b25eff" + FEAT_NFT: "true" + NFT_FEATURE_START_BLOCK: 4050733 + FEAT_WRAP_UNWRAP_NATIVE_TRANSFER_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' event-pipeline-base: + depends_on: + - postgres build: context: . dockerfile: Dockerfile.dev restart: always environment: - EVM_RPC_URL: "${RPC_URL_BASE}" # Use a secret or .env file for sensitive data - CHAIN_ID: "8453" - SCHEMA: "events_base" - LOG_LEVEL: "info" - SCRAPER_MODE: "BLOCKS" - BLOCK_RECEIPTS_MODE: "alchemy" - ENABLE_PROMETHEUS_METRICS: "true" - SECONDS_BETWEEN_RUNS: "1" - RESCRAPE_BLOCKS: "10" - FEAT_ZEROEX_EXCHANGE_PROXY: "true" - EP_DEPLOYMENT_BLOCK: "1410394" - SETTLER_DEPLOYMENT_BLOCK: "12723120" - MAX_BLOCKS_TO_SEARCH: "500" - MAX_BLOCKS_TO_PULL: "102" - MAX_BLOCKS_REORG: "300" - FEAT_TRANSFORMED_ERC20_EVENT: "true" - FEAT_ERC20_BRIDGE_TRANSFER_FLASHWALLET: "true" - FLASHWALLET_ADDRESS: "0xdb6f1920a889355780af7570773609bd8cb1f498" - FLASHWALLET_DEPLOYMENT_BLOCK: "1410394" + ETHEREUM_RPC_URL: '${RPC_URL_BASE}' + CHAIN_ID: '8453' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events_base' + EP_DEPLOYMENT_BLOCK: 1410394 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 100 + SECONDS_BETWEEN_RUNS: 1 FEAT_NFT: "true" - NFT_FEATURE_START_BLOCK: "1410394" - FEAT_OTC_ORDERS: "true" - OTC_ORDERS_FEATURE_START_BLOCK: "1410394" - BLOCK_FINALITY_THRESHOLD: "0" # No longer used + NFT_FEATURE_START_BLOCK: 1410394 + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: "0x4200000000000000000000000000000000000006" - FEAT_SOCKET_BRIDGE_EVENT: "true" - SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" - SOCKET_BRIDGE_EVENT_START_BLOCK: "6003127" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4200000000000000000000000000000000000006' + + event-pipeline-scroll: + build: + context: . + dockerfile: Dockerfile.dev + platform: linux/amd64 + restart: always + environment: + SCRAPER_MODE: 'BLOCKS' + EVM_RPC_URL: '${RPC_URL_SCROLL}' + CHAIN_ID: '534352' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' + SCHEMA: 'events_scroll' + FEAT_ZEROEX_EXCHANGE_PROXY: "false" + SETTLER_DEPLOYMENT_BLOCK: 7474150 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 100 + SECONDS_BETWEEN_RUNS: 1 + RESCRAPE_BLOCKS: 10 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x5300000000000000000000000000000000000004' FEAT_TOKENS_FROM_TRANSFERS: "true" - FEAT_META_TRANSACTION_EXECUTED_EVENT: "true" - META_TRANSACTION_EXECUTED_START_BLOCK: "10540349" FEAT_ERC20_TRANSFER_ALL: "true" FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - FEAT_SETTLER_RFQ_ORDER_EVENT: "false" - POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + FEAT_SETTLER_RFQ_ORDER_EVENT: "true" networks: - 0x-data-migrations_default - # event-pipeline-scroll: - # build: - # context: . - # dockerfile: Dockerfile.dev - # platform: linux/amd64 - # restart: always - # environment: - # SCRAPER_MODE: 'BLOCKS' - # EVM_RPC_URL: '${RPC_URL_SCROLL}' - # CHAIN_ID: '534352' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' - # SCHEMA: 'events_scroll' - # FEAT_ZEROEX_EXCHANGE_PROXY: "false" - # SETTLER_DEPLOYMENT_BLOCK: 7474150 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 100 - # SECONDS_BETWEEN_RUNS: 1 - # RESCRAPE_BLOCKS: 10 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x5300000000000000000000000000000000000004' - # FEAT_TOKENS_FROM_TRANSFERS: "true" - # FEAT_ERC20_TRANSFER_ALL: "true" - # FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - # FEAT_SETTLER_RFQ_ORDER_EVENT: "true" - # networks: - # - 0x-data-migrations_default - - # event-pipeline-blast: - # build: - # context: . - # dockerfile: Dockerfile.dev - # platform: linux/amd64 - # restart: always - # environment: - # SCRAPER_MODE: 'EVENTS' - # EVM_RPC_URL: '${RPC_URL_BLAST}' - # CHAIN_ID: '81457' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' - # SCHEMA: 'events_blast' - # FEAT_ZEROEX_EXCHANGE_PROXY: "false" - # SETTLER_DEPLOYMENT_BLOCK: 6124420 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 100 - # SECONDS_BETWEEN_RUNS: 1 - # RESCRAPE_BLOCKS: 10 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4300000000000000000000000000000000000004' - # FEAT_TOKENS_FROM_TRANSFERS: "true" - # FEAT_ERC20_TRANSFER_ALL: "true" - # FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - # TOKENS_FROM_TRANSFERS_START_BLOCK: "1" - # SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" - # SOCKET_BRIDGE_EVENT_START_BLOCK: "6124420" - # networks: - # - 0x-data-migrations_default + event-pipeline-blast: + build: + context: . + dockerfile: Dockerfile.dev + platform: linux/amd64 + restart: always + environment: + SCRAPER_MODE: 'EVENTS' + EVM_RPC_URL: '${RPC_URL_BLAST}' + CHAIN_ID: '81457' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' + SCHEMA: 'events_blast' + FEAT_ZEROEX_EXCHANGE_PROXY: "false" + SETTLER_DEPLOYMENT_BLOCK: 6124420 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 100 + SECONDS_BETWEEN_RUNS: 1 + RESCRAPE_BLOCKS: 10 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x4300000000000000000000000000000000000004' + FEAT_TOKENS_FROM_TRANSFERS: "true" + FEAT_ERC20_TRANSFER_ALL: "true" + FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" + TOKENS_FROM_TRANSFERS_START_BLOCK: "1" + SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" + SOCKET_BRIDGE_EVENT_START_BLOCK: "6124420" + networks: + - 0x-data-migrations_default - # event-pipeline-linea: - # build: - # context: . - # dockerfile: Dockerfile.dev - # platform: linux/amd64 - # restart: always - # environment: - # SCRAPER_MODE: 'EVENTS' - # EVM_RPC_URL: '${RPC_URL_LINEA}' - # CHAIN_ID: '59144' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' - # SCHEMA: 'events_linea' - # FEAT_ZEROEX_EXCHANGE_PROXY: "false" - # SETTLER_DEPLOYMENT_BLOCK: 6917652 - # MAX_BLOCKS_TO_SEARCH: 125 - # MAX_BLOCKS_TO_PULL: 25 - # MAX_BLOCKS_REORG: 125 - # SECONDS_BETWEEN_RUNS: 1 - # RESCRAPE_BLOCKS: 10 - # FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" - # WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xe5d7c2a44ffddf6b295a15c148167daaaf5cf34f' - # FEAT_TOKENS_FROM_TRANSFERS: "true" - # FEAT_ERC20_TRANSFER_ALL: "true" - # FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" - # TOKENS_FROM_TRANSFERS_START_BLOCK: "1" - # FEAT_SOCKET_BRIDGE_EVENT: "true" - # SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" - # SOCKET_BRIDGE_EVENT_START_BLOCK: "6917652" - # networks: - # - 0x-data-migrations_default + event-pipeline-linea: + build: + context: . + dockerfile: Dockerfile.dev + platform: linux/amd64 + restart: always + environment: + SCRAPER_MODE: 'EVENTS' + EVM_RPC_URL: '${RPC_URL_LINEA}' + CHAIN_ID: '59144' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}' + SCHEMA: 'events_linea' + FEAT_ZEROEX_EXCHANGE_PROXY: "false" + SETTLER_DEPLOYMENT_BLOCK: 6917652 + MAX_BLOCKS_TO_SEARCH: 125 + MAX_BLOCKS_TO_PULL: 25 + MAX_BLOCKS_REORG: 125 + SECONDS_BETWEEN_RUNS: 1 + RESCRAPE_BLOCKS: 10 + FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true" + WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xe5d7c2a44ffddf6b295a15c148167daaaf5cf34f' + FEAT_TOKENS_FROM_TRANSFERS: "true" + FEAT_ERC20_TRANSFER_ALL: "true" + FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true" + TOKENS_FROM_TRANSFERS_START_BLOCK: "1" + FEAT_SOCKET_BRIDGE_EVENT: "true" + SOCKET_BRIDGE_CONTRACT_ADDRESS: "0x3a23f943181408eac424116af7b7790c94cb97a5" + SOCKET_BRIDGE_EVENT_START_BLOCK: "6917652" + networks: + - 0x-data-migrations_default - # token-scraper-ethereum: - # depends_on: - # - postgres - # build: - # context: . - # dockerfile: Dockerfile.dev - # restart: always - # environment: - # ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' - # CHAIN_ID: '1' - # POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' - # SCHEMA: 'events' - # KAFKA_BROKERS: '${KAFKA_BROKERS}' - # KAFKA_SSL: '${KAFKA_SSL}' - # KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' - # KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' - # LOG_LEVEL: "info" - # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" - # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 - # EP_DEPLOYMENT_BLOCK: 10247094 - # MAX_BLOCKS_TO_SEARCH: 1000 - # MAX_BLOCKS_TO_PULL: 1000 - # MAX_TX_TO_PULL: 1000 - # SECONDS_BETWEEN_RUNS: 5 + token-scraper-ethereum: + depends_on: + - postgres + build: + context: . + dockerfile: Dockerfile.dev + restart: always + environment: + ETHEREUM_RPC_URL: '${RPC_URL_ETHEREUM}' + CHAIN_ID: '1' + POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}' + SCHEMA: 'events' + KAFKA_BROKERS: '${KAFKA_BROKERS}' + KAFKA_SSL: '${KAFKA_SSL}' + KAFKA_AUTH_USER: '${KAFKA_AUTH_USER}' + KAFKA_AUTH_PASSWORD: '${KAFKA_AUTH_PASSWORD}' + LOG_LEVEL: "info" + FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true" + TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266 + EP_DEPLOYMENT_BLOCK: 10247094 + MAX_BLOCKS_TO_SEARCH: 1000 + MAX_BLOCKS_TO_PULL: 1000 + MAX_TX_TO_PULL: 1000 + SECONDS_BETWEEN_RUNS: 5 networks: 0x-data-migrations_default: From 617c8ee987a46908797b2a3807cebbaa58872eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Elizondo?= Date: Fri, 8 Nov 2024 06:14:29 -0600 Subject: [PATCH 7/9] Changes wording to requiredTxnList, as it forces these txns and allows any others. --- src/events.ts | 4 ++-- src/filters/erc20_transfer_events.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/events.ts b/src/events.ts index 82089e2e..b1615424 100644 --- a/src/events.ts +++ b/src/events.ts @@ -265,11 +265,11 @@ export type EventScraperProps = { deleteOptions?: DeleteOptions; tokenMetadataMap?: TokenMetadataMap; postProcess?: any; - filterFunction?: (events: Event[], transaction: Transaction, allowedTxnList?: Set) => Event[]; + filterFunction?: (events: Event[], transaction: Transaction, requiredTxnList?: Set) => Event[]; filterFunctionGetContext?: ( events: Event[], web3Source: Web3Source, - allowedTxnList?: Set, + requiredTxnList?: Set, ) => Promise; }; diff --git a/src/filters/erc20_transfer_events.ts b/src/filters/erc20_transfer_events.ts index 5f02e39a..47b09313 100644 --- a/src/filters/erc20_transfer_events.ts +++ b/src/filters/erc20_transfer_events.ts @@ -5,13 +5,13 @@ import { getParseTxsAsync } from '../scripts/utils/web3_utils'; export async function filterERC20TransferEventsGetContext( events: Event[], web3Source: Web3Source, - allowedTxnList?: Set, + requiredTxnList?: Set, ): Promise { if (events.length > 0) { const txHashes = events.map((log: Event) => log.transactionHash); let validTxHashSet: Set; - if (allowedTxnList && allowedTxnList.size > 0) { - validTxHashSet = allowedTxnList; + if (requiredTxnList && requiredTxnList.size > 0) { + validTxHashSet = requiredTxnList; } else { const txData = await getParseTxsAsync(web3Source, txHashes); const filteredTxsHashes = txData.parsedTxs @@ -30,11 +30,11 @@ export async function filterERC20TransferEventsGetContext( export function filterERC20TransferEvents( events: Event[], transaction: Transaction, - allowedTxnList?: Set, + requiredTxnList?: Set, ): Event[] { const filteredEvents = new Set(); - if (allowedTxnList && allowedTxnList.size > 0 && allowedTxnList.has(transaction.transactionHash)) { + if (requiredTxnList && requiredTxnList.size > 0 && requiredTxnList.has(transaction.transactionHash)) { events.filter((e) => e !== null).forEach((e) => filteredEvents.add(e)); } From cf35c3a071f023cde9255d5b518e7fcf3d8ad4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Elizondo?= Date: Fri, 8 Nov 2024 06:17:48 -0600 Subject: [PATCH 8/9] Changes wording to requiredTxnList in remaining file. --- src/scripts/pull_and_save_block_events.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index d82cb95d..99183142 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -73,12 +73,12 @@ const provider = web3Factory.getRpcProvider({ }); const web3Source = new Web3Source(provider, EVM_RPC_URL); -function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set): ParsedFullBlock { +function parseBlockTransactionsEvents(fullBlock: FullBlock, requiredTxnList?: Set): ParsedFullBlock { const parsedBlock = parseBlock({ ...fullBlock, transactions: [''] }); const usefullTxs: ParsedTransaction[] = fullBlock.transactions .map((transaction: FullTransaction): ParsedTransaction | null => { - const parsedTransactionEvents = parseTransactionEvents(transaction, allowedTxnList); + const parsedTransactionEvents = parseTransactionEvents(transaction, requiredTxnList); if (parsedTransactionEvents.parsedTransaction !== null) { return parsedTransactionEvents; } @@ -101,7 +101,7 @@ function parseBlockTransactionsEvents(fullBlock: FullBlock, allowedTxnList?: Set }; } -function parseTransactionEvents(transaction: FullTransaction, allowedTxnList?: Set): ParsedTransaction { +function parseTransactionEvents(transaction: FullTransaction, requiredTxnList?: Set): ParsedTransaction { if (transaction.input === '0x') { return { parsedTransaction: null, @@ -121,7 +121,7 @@ function parseTransactionEvents(transaction: FullTransaction, allowedTxnList?: S const parsedLogs = baseFilteredLogs.map((log: LogEntry) => props.parser(log)); const filteredLogs = props.filterFunction - ? props.filterFunction(parsedLogs, parsedTransaction, allowedTxnList) + ? props.filterFunction(parsedLogs, parsedTransaction, requiredTxnList) : parsedLogs; const postProcessedLogs = props.postProcess ? props.postProcess(filteredLogs) : filteredLogs; @@ -269,7 +269,7 @@ async function getParseSaveBlocksTransactionsEvents( producer: Producer | null, newBlocks: EVMBlock[], allowPartialSuccess: boolean, - allowedTxnList?: Set, + requiredTxnList?: Set, ): Promise { const blockNumbers = newBlocks.map((newBlock) => newBlock.number!); @@ -327,7 +327,7 @@ async function getParseSaveBlocksTransactionsEvents( return { ...newBlocks[blockIndex], transactions: transactionsWithLogs }; }); - const parsedFullBlocks = fullBlocks.map((fullBlock) => parseBlockTransactionsEvents(fullBlock, allowedTxnList)); + const parsedFullBlocks = fullBlocks.map((fullBlock) => parseBlockTransactionsEvents(fullBlock, requiredTxnList)); const eventTables = eventScrperProps .filter((props) => props.enabled) @@ -371,14 +371,14 @@ export class BlockEventsScraper { const blockNumbers = oldestBlocksToBackfill.map( (backfillBlock: { block_number: number }) => backfillBlock.block_number, ); - const allowedTxnListQuery = await connection.query( + const requiredTxnListQuery = await connection.query( `SELECT DISTINCT transaction_hash FROM ${SCHEMA}.tx_backfill WHERE block_number IN (${blockNumbers.join(',')}) AND done = false`, ); - const allowedTxnList = new Set( - allowedTxnListQuery.map((row: { transaction_hash: string }) => row.transaction_hash), + const requiredTxnList = new Set( + requiredTxnListQuery.map((row: { transaction_hash: string }) => row.transaction_hash), ); const newBlocks = await web3Source.getBatchBlockInfoAsync(blockNumbers, true); @@ -388,7 +388,7 @@ export class BlockEventsScraper { producer, newBlocks, false, - allowedTxnList, + requiredTxnList, ); if (success) { From 95a0f4ec3dad665f7b1778f743e8c357dc4b5512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Elizondo?= Date: Fri, 8 Nov 2024 06:23:01 -0600 Subject: [PATCH 9/9] Prettier. --- src/scripts/pull_and_save_block_events.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index 99183142..af005c6a 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -327,7 +327,9 @@ async function getParseSaveBlocksTransactionsEvents( return { ...newBlocks[blockIndex], transactions: transactionsWithLogs }; }); - const parsedFullBlocks = fullBlocks.map((fullBlock) => parseBlockTransactionsEvents(fullBlock, requiredTxnList)); + const parsedFullBlocks = fullBlocks.map((fullBlock) => + parseBlockTransactionsEvents(fullBlock, requiredTxnList), + ); const eventTables = eventScrperProps .filter((props) => props.enabled)