diff --git a/src/data_sources/events/web3.ts b/src/data_sources/events/web3.ts index 6faf3256..43691dd4 100644 --- a/src/data_sources/events/web3.ts +++ b/src/data_sources/events/web3.ts @@ -1,21 +1,21 @@ import { MAX_TX_TO_PULL, BLOCK_RECEIPTS_MODE } from '../../config'; -import { BLOCK_RECEIPTS_MODE_ENDPOINT } from '../../constants'; import { chunk, logger } from '../../utils'; -import { Web3ProviderEngine } from '@0x/subproviders'; -import { Web3Wrapper } from '@0x/web3-wrapper'; import { BlockWithTransactionData, BlockWithoutTransactionData, - RawLog, + alchemyBlockTransactionReceiptsFormatter, + updatedBlockFormatter, + outputTransactionReceiptFormatter, Transaction, TransactionReceipt, -} from 'ethereum-types'; +} from './web3_updated'; +import { Web3ProviderEngine } from '@0x/subproviders'; +import { Web3Wrapper } from '@0x/web3-wrapper'; +import { RawLog } from 'ethereum-types'; const Web3 = require('web3'); -const utils = require('web3-utils'); -const helpers = require('web3-core-helpers'); -const formatter = helpers.formatters; +export { BlockWithTransactionData, BlockWithoutTransactionData, Transaction, TransactionReceipt } from './web3_updated'; export interface LogPullInfo { address: string | null; @@ -28,47 +28,6 @@ export interface ContractCallInfo { data: string; } -export interface Transaction1559 extends Transaction { - type: number; -} - -export interface BlockWithoutTransactionData1559 extends BlockWithoutTransactionData { - baseFeePerGas: number; -} -export interface BlockWithTransactionData1559 extends BlockWithTransactionData { - baseFeePerGas: number; - transactions: Transaction1559[]; -} - -export interface TransactionReceipt1559 extends TransactionReceipt { - effectiveGasPrice: number; -} - -const alchemyBlockTransactionReceiptsFormatter = function (response: any): TransactionReceipt1559[] { - if (typeof response !== 'object') { - throw new Error('Received receipt is invalid: ' + response); - } - - return response.receipts.map((receipt: any) => { - if (receipt.blockNumber !== null) receipt.blockNumber = utils.hexToNumber(receipt.blockNumber); - if (receipt.transactionIndex !== null) receipt.transactionIndex = utils.hexToNumber(receipt.transactionIndex); - receipt.cumulativeGasUsed = utils.hexToNumber(receipt.cumulativeGasUsed); - receipt.gasUsed = utils.hexToNumber(receipt.gasUsed); - if (receipt.effectiveGasPrice) { - receipt.effectiveGasPrice = utils.hexToNumber(receipt.effectiveGasPrice); - } - if (Array.isArray(receipt.logs)) { - receipt.logs = receipt.logs.map(formatter.outputLogFormatter); - } - - if (typeof receipt.status !== 'undefined' && receipt.status !== null) { - receipt.status = Boolean(parseInt(receipt.status)); - } - - return receipt; - }); -}; - export class Web3Source { private readonly _web3Wrapper: Web3Wrapper; private readonly _web3: any; @@ -84,7 +43,7 @@ export class Web3Source { call: 'eth_getBlockReceipts', params: 1, inputFormatter: [this._web3.utils.numberToHex], - outputFormatter: (block: any) => formatter.outputTransactionReceiptFormatter(block), + outputFormatter: outputTransactionReceiptFormatter, }, ], }); @@ -103,19 +62,30 @@ export class Web3Source { ], }); } + this._web3.eth.extend({ + methods: [ + { + name: 'getBlockByNumberN', + call: 'eth_getBlockByNumber', + params: 2, + inputFormatter: [this._web3.utils.numberToHex, null], + outputFormatter: updatedBlockFormatter, + }, + ], + }); } public async getBatchBlockInfoForRangeAsync( startBlock: number, endBlock: number, includeTransactions: B, - ): Promise; + ): Promise; public async getBatchBlockInfoForRangeAsync( startBlock: number, endBlock: number, includeTransactions: boolean, - ): Promise<(BlockWithoutTransactionData | BlockWithTransactionData1559)[]> { + ): Promise<(BlockWithoutTransactionData | BlockWithTransactionData)[]> { const blockNumbers = Array.from(Array(endBlock - startBlock + 1).keys()).map((i) => i + startBlock); return this.getBatchBlockInfoAsync(blockNumbers, includeTransactions); } @@ -123,20 +93,20 @@ export class Web3Source { public async getBatchBlockInfoAsync( blockNumbers: number[], includeTransactions: B, - ): Promise; + ): Promise; public async getBatchBlockInfoAsync( blockNumbers: number[], includeTransactions: boolean, - ): Promise<(BlockWithoutTransactionData | BlockWithTransactionData1559)[]> { + ): Promise<(BlockWithoutTransactionData | BlockWithTransactionData)[]> { const batch = new this._web3.BatchRequest(); const promises = blockNumbers.map((blockNumber) => { - return new Promise((resolve, reject) => { - const req = this._web3.eth.getBlock.request( + return new Promise((resolve, reject) => { + const req = this._web3.eth.getBlockByNumberN.request( blockNumber, includeTransactions, - (err: any, data: BlockWithTransactionData1559) => { + (err: any, data: BlockWithTransactionData) => { if (err) { logger.error(`Blocks error: ${err}`); reject(err); @@ -157,12 +127,12 @@ export class Web3Source { public async getBatchBlockReceiptsForRangeAsync( startBlock: number, endBlock: number, - ): Promise { + ): Promise { const blockNumbers = Array.from(Array(endBlock - startBlock + 1).keys()).map((i) => i + startBlock); return this.getBatchBlockReceiptsAsync(blockNumbers); } - public async getBatchBlockReceiptsAsync(blockNumbers: number[]): Promise { + public async getBatchBlockReceiptsAsync(blockNumbers: number[]): Promise { const promises = blockNumbers.map((blockNumber) => { return this._web3.eth.getBlockReceipts(blockNumber).catch((err: any) => { logger.error(`Blocks error: ${err}`); @@ -286,7 +256,7 @@ export class Web3Source { try { logger.debug(`Fetching block ${blockNumber}`); - const block = await this._web3Wrapper.getBlockIfExistsAsync(blockNumber); + const block = (await this._web3Wrapper.getBlockIfExistsAsync(blockNumber)) as BlockWithoutTransactionData; if (block == null) { throw new Error(`Block ${blockNumber} returned null`); @@ -303,7 +273,7 @@ export class Web3Source { } public async getTransactionInfoAsync(txHash: string): Promise { - return this._web3Wrapper.getTransactionByHashAsync(txHash); + return (await this._web3Wrapper.getTransactionByHashAsync(txHash)) as Transaction; } public async getBlockNumberAsync(): Promise { diff --git a/src/data_sources/events/web3_updated.ts b/src/data_sources/events/web3_updated.ts new file mode 100644 index 00000000..613cf6d6 --- /dev/null +++ b/src/data_sources/events/web3_updated.ts @@ -0,0 +1,108 @@ +import { + Transaction as TransactionOld, + BlockWithoutTransactionData as BlockWithoutTransactionDataOld, + BlockWithTransactionData as BlockWithTransactionDataOld, + TransactionReceipt as TransactionReceiptOld, +} from 'ethereum-types'; + +const utils = require('web3-utils'); +const helpers = require('web3-core-helpers'); +const formatter = helpers.formatters; + +export interface Transaction extends TransactionOld { + type: number; +} + +export interface BlockWithoutTransactionData extends BlockWithoutTransactionDataOld { + baseFeePerGas: number; +} +export interface BlockWithTransactionData extends BlockWithTransactionDataOld { + baseFeePerGas: number; + transactions: Transaction[]; +} + +export interface TransactionReceipt extends TransactionReceiptOld { + effectiveGasPrice: number; +} + +export const outputTransactionReceiptFormatter = (raw: any) => formatter.outputTransactionReceiptFormatter(raw); + +export const alchemyBlockTransactionReceiptsFormatter = function (response: any): TransactionReceipt[] { + if (typeof response !== 'object') { + throw new Error('Received receipt is invalid: ' + response); + } + + return response.receipts.map((receipt: any) => { + if (receipt.blockNumber !== null) receipt.blockNumber = utils.hexToNumber(receipt.blockNumber); + if (receipt.transactionIndex !== null) receipt.transactionIndex = utils.hexToNumber(receipt.transactionIndex); + receipt.cumulativeGasUsed = utils.hexToNumber(receipt.cumulativeGasUsed); + receipt.gasUsed = utils.hexToNumber(receipt.gasUsed); + if (receipt.effectiveGasPrice) { + receipt.effectiveGasPrice = utils.hexToNumber(receipt.effectiveGasPrice); + } + if (Array.isArray(receipt.logs)) { + receipt.logs = receipt.logs.map(formatter.outputLogFormatter); + } + + if (typeof receipt.status !== 'undefined' && receipt.status !== null) { + receipt.status = Boolean(parseInt(receipt.status)); + } + + return receipt; + }); +}; + +export const outputBigNumberFormatter = function (number: number) { + return utils.toBN(number).toString(10); +}; + +export const updatedTransactionFormater = function (tx: any): Transaction { + if (tx.blockNumber !== null) tx.blockNumber = utils.hexToNumber(tx.blockNumber); + if (tx.transactionIndex !== null) tx.transactionIndex = utils.hexToNumber(tx.transactionIndex); + tx.nonce = utils.hexToNumber(tx.nonce); + tx.gas = outputBigNumberFormatter(tx.gas); + if (tx.type) tx.type = utils.hexToNumber(tx.type); + if (tx.gasPrice) tx.gasPrice = outputBigNumberFormatter(tx.gasPrice); + if (tx.maxFeePerGas) tx.maxFeePerGas = outputBigNumberFormatter(tx.maxFeePerGas); + if (tx.maxPriorityFeePerGas) tx.maxPriorityFeePerGas = outputBigNumberFormatter(tx.maxPriorityFeePerGas); + tx.value = outputBigNumberFormatter(tx.value); + + if (tx.to && utils.isAddress(tx.to)) { + // tx.to could be `0x0` or `null` while contract creation + tx.to = utils.toChecksumAddress(tx.to); + } else { + tx.to = null; // set to `null` if invalid address + } + + if (tx.from) { + tx.from = utils.toChecksumAddress(tx.from); + } + + return tx; +}; +export const updatedBlockFormatter = function (block: any): BlockWithTransactionData | BlockWithoutTransactionData { + // transform to number + block.gasLimit = utils.hexToNumber(block.gasLimit); + block.gasUsed = utils.hexToNumber(block.gasUsed); + block.size = utils.hexToNumber(block.size); + block.timestamp = utils.hexToNumber(block.timestamp); + if (block.number !== null) block.number = utils.hexToNumber(block.number); + + if (block.difficulty) block.difficulty = outputBigNumberFormatter(block.difficulty); + if (block.totalDifficulty) block.totalDifficulty = outputBigNumberFormatter(block.totalDifficulty); + + if (Array.isArray(block.transactions)) { + block.transactions.forEach(function (item: any) { + if (!(typeof item === 'string')) { + return updatedTransactionFormater(item); + } + return item; + }); + } + + if (block.miner) block.miner = utils.toChecksumAddress(block.miner); + + if (block.baseFeePerGas) block.baseFeePerGas = utils.hexToNumber(block.baseFeePerGas); + + return block; +}; diff --git a/src/parsers/web3/parse_web3_objects.ts b/src/parsers/web3/parse_web3_objects.ts index e79fe076..4711af3b 100644 --- a/src/parsers/web3/parse_web3_objects.ts +++ b/src/parsers/web3/parse_web3_objects.ts @@ -1,8 +1,8 @@ import { ZEROEX_API_AFFILIATE_SELECTOR } from '../../constants'; import { - BlockWithoutTransactionData1559, - TransactionReceipt1559 as RawReceipt, - Transaction1559 as EVMTransaction, + BlockWithoutTransactionData, + TransactionReceipt as RawReceipt, + Transaction as EVMTransaction, } from '../../data_sources/events/web3'; import { Block, Transaction, TransactionLogs, TransactionReceipt } from '../../entities'; import { BigNumber } from '@0x/utils'; @@ -93,7 +93,7 @@ export function parseTransactionLogs(rawReceipt: RawReceipt): TransactionLogs { * Converts a raw block into a Block entity * @param rawBlock Raw block without transaction info returned from JSON RPC */ -export function parseBlock(rawBlock: BlockWithoutTransactionData1559): Block { +export function parseBlock(rawBlock: BlockWithoutTransactionData): Block { const parsedBlock = new Block(); parsedBlock.observedTimestamp = new Date().getTime(); diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts index 4e713e83..6de35bae 100644 --- a/src/scripts/pull_and_save_block_events.ts +++ b/src/scripts/pull_and_save_block_events.ts @@ -8,10 +8,10 @@ import { SCHEMA, } from '../config'; import { TRANSFER_EVENT_TOPIC_0 } from '../constants'; -import { Web3Source, BlockWithTransactionData1559 as EVMBlock } from '../data_sources/events/web3'; +import { Web3Source, BlockWithTransactionData as EVMBlock } from '../data_sources/events/web3'; import { - Transaction1559 as EVMTransaction, - TransactionReceipt1559 as EVMTransactionReceipt, + Transaction as EVMTransaction, + TransactionReceipt as EVMTransactionReceipt, } from '../data_sources/events/web3'; import { Block, Transaction, TransactionReceipt } from '../entities'; import { eventScrperProps, EventScraperProps } from '../events';