From b0d608844d614e09d5c042d9991f1573f62e91f6 Mon Sep 17 00:00:00 2001 From: iuwqyir Date: Tue, 15 Oct 2024 13:06:50 +0300 Subject: [PATCH] utilize eth_getBlockReceipts if possible --- README.md | 37 ++++ cmd/root.go | 6 + configs/config.example.yml | 9 + configs/config.go | 18 +- internal/common/log.go | 2 + internal/common/transaction.go | 10 +- internal/rpc/params.go | 4 + internal/rpc/rpc.go | 108 ++++++++--- internal/rpc/serializer.go | 173 +++++++++++++++--- internal/rpc/utils.go | 19 +- internal/storage/clickhouse.go | 10 +- .../clickhouse_create_transactions_table.sql | 10 +- 12 files changed, 337 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index b90d7e1..9a45eeb 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,43 @@ rpc: batchDelay: 100 ``` +#### RPC Block Receipts Enabled +If this is `true`, will use `eth_getBlockReceipts` instead of `eth_getLogs` if the RPC supports it. Allows getting receipt data for transactions, but is not supported by every RPC. Default is `false`. + +cmd: `--rpc-block-receipts-enabled` +env: `RPC_BLOCKRECEIPTS_ENABLED` +yaml: +```yaml +rpc: + blockReceipts: + enabled: true +``` + +#### RPC Block Receipts Blocks Per Request +How many blocks at a time to fetch block receipts for from the RPC. Default is 250. +Has no effect if it's larger than RPC blocks per request. + +cmd: `--rpc-block-receipts-blocksPerRequest` +env: `RPC_BLOCKRECEIPTS_BLOCKSPERREQUEST` +yaml: +```yaml +rpc: + blockReceipts: + blocksPerRequest: 100 +``` + +#### RPC Block Receipts Batch Delay +Milliseconds to wait between batches of block receipts when fetching from the RPC. Default is 0. + +cmd: `--rpc-block-receipts-batchDelay` +env: `RPC_BLOCKRECEIPTS_BATCHDELAY` +yaml: +```yaml +rpc: + blockReceipts: + batchDelay: 100 +``` + #### RPC Traces Enabled Whether to enable fetching traces from the RPC. Default is `true`, but it will try to detect if the RPC supports traces automatically. diff --git a/cmd/root.go b/cmd/root.go index 6503886..fe310ab 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -41,6 +41,9 @@ func init() { rootCmd.PersistentFlags().Int("rpc-blocks-batchDelay", 0, "Milliseconds to wait between batches of blocks when fetching from the RPC") rootCmd.PersistentFlags().Int("rpc-logs-blocksPerRequest", 0, "How many blocks to fetch logs per request") rootCmd.PersistentFlags().Int("rpc-logs-batchDelay", 0, "Milliseconds to wait between batches of logs when fetching from the RPC") + rootCmd.PersistentFlags().Bool("rpc-blockReceipts-enabled", false, "Whether to enable fetching block receipts from the RPC") + rootCmd.PersistentFlags().Int("rpc-blockReceipts-blocksPerRequest", 0, "How many blocks to fetch receipts for per request") + rootCmd.PersistentFlags().Int("rpc-blockReceipts-batchDelay", 0, "Milliseconds to wait between batches of receipts when fetching from the RPC") rootCmd.PersistentFlags().Bool("rpc-traces-enabled", true, "Whether to enable fetching traces from the RPC") rootCmd.PersistentFlags().Int("rpc-traces-blocksPerRequest", 0, "How many blocks to fetch traces per request") rootCmd.PersistentFlags().Int("rpc-traces-batchDelay", 0, "Milliseconds to wait between batches of traces when fetching from the RPC") @@ -89,6 +92,9 @@ func init() { viper.BindPFlag("rpc.blocks.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-blocks-batchDelay")) viper.BindPFlag("rpc.logs.blocksPerRequest", rootCmd.PersistentFlags().Lookup("rpc-logs-blocksPerRequest")) viper.BindPFlag("rpc.logs.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-logs-batchDelay")) + viper.BindPFlag("rpc.blockReceipts.enabled", rootCmd.PersistentFlags().Lookup("rpc-blockReceipts-enabled")) + viper.BindPFlag("rpc.blockReceipts.blocksPerRequest", rootCmd.PersistentFlags().Lookup("rpc-blockReceipts-blocksPerRequest")) + viper.BindPFlag("rpc.blockReceipts.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-blockReceipts-batchDelay")) viper.BindPFlag("rpc.traces.enabled", rootCmd.PersistentFlags().Lookup("rpc-traces-enabled")) viper.BindPFlag("rpc.traces.blocksPerRequest", rootCmd.PersistentFlags().Lookup("rpc-traces-blocksPerRequest")) viper.BindPFlag("rpc.traces.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-traces-batchDelay")) diff --git a/configs/config.example.yml b/configs/config.example.yml index 7fc79b1..0c3eb9b 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -5,6 +5,10 @@ rpc: logs: blocksPerRequest: 400 batchDelay: 100 + blockReceipts: + enabled: true + blocksPerRequest: 500 + batchDelay: 100 traces: enabled: true blocksPerRequest: 200 @@ -29,6 +33,11 @@ failureRecoverer: interval: 10000 blocksPerRun: 100 +reorgHandler: + enabled: true + interval: 1000 + blocksPerScan: 50 + storage: main: clickhouse: diff --git a/configs/config.go b/configs/config.go index d92ae95..091146a 100644 --- a/configs/config.go +++ b/configs/config.go @@ -81,22 +81,22 @@ type RedisConfig struct { DB int `mapstructure:"db"` } -type RPCBatchSizeConfig struct { +type RPCBatchRequestConfig struct { BlocksPerRequest int `mapstructure:"blocksPerRequest"` BatchDelay int `mapstructure:"batchDelay"` } -type RPCTracesConfig struct { - Enabled bool `mapstructure:"enabled"` - BlocksPerRequest int `mapstructure:"blocksPerRequest"` - BatchDelay int `mapstructure:"batchDelay"` +type ToggleableRPCBatchRequestConfig struct { + Enabled bool `mapstructure:"enabled"` + RPCBatchRequestConfig } type RPCConfig struct { - URL string `mapstructure:"url"` - Blocks RPCBatchSizeConfig `mapstructure:"blocks"` - Logs RPCBatchSizeConfig `mapstructure:"logs"` - Traces RPCTracesConfig `mapstructure:"traces"` + URL string `mapstructure:"url"` + Blocks RPCBatchRequestConfig `mapstructure:"blocks"` + Logs RPCBatchRequestConfig `mapstructure:"logs"` + BlockReceipts ToggleableRPCBatchRequestConfig `mapstructure:"blockReceipts"` + Traces ToggleableRPCBatchRequestConfig `mapstructure:"traces"` } type APIConfig struct { diff --git a/internal/common/log.go b/internal/common/log.go index ca83b4c..d23ddfe 100644 --- a/internal/common/log.go +++ b/internal/common/log.go @@ -18,3 +18,5 @@ type Log struct { } type RawLogs = []map[string]interface{} +type RawReceipts = []RawReceipt +type RawReceipt = map[string]interface{} diff --git a/internal/common/transaction.go b/internal/common/transaction.go index 2464c11..3c4afd9 100644 --- a/internal/common/transaction.go +++ b/internal/common/transaction.go @@ -25,5 +25,13 @@ type Transaction struct { R *big.Int `json:"r"` S *big.Int `json:"s"` V *big.Int `json:"v"` - AccessListJson string `json:"access_list_json"` + AccessListJson *string `json:"access_list_json"` + ContractAddress *string `json:"contract_address"` + GasUsed *uint64 `json:"gas_used"` + CumulativeGasUsed *uint64 `json:"cumulative_gas_used"` + EffectiveGasPrice *big.Int `json:"effective_gas_price"` + BlobGasUsed *uint64 `json:"blob_gas_used"` + BlobGasPrice *big.Int `json:"blob_gas_price"` + LogsBloom *string `json:"logs_bloom"` + Status *uint64 `json:"status"` } diff --git a/internal/rpc/params.go b/internal/rpc/params.go index 9dc65e9..220f996 100644 --- a/internal/rpc/params.go +++ b/internal/rpc/params.go @@ -21,3 +21,7 @@ func GetLogsParams(blockNum *big.Int) []interface{} { func TraceBlockParams(blockNum *big.Int) []interface{} { return []interface{}{hexutil.EncodeBig(blockNum)} } + +func GetBlockReceiptsParams(blockNum *big.Int) []interface{} { + return []interface{}{hexutil.EncodeBig(blockNum)} +} diff --git a/internal/rpc/rpc.go b/internal/rpc/rpc.go index b2186a3..8638fd0 100644 --- a/internal/rpc/rpc.go +++ b/internal/rpc/rpc.go @@ -27,9 +27,10 @@ type GetBlocksResult struct { } type BlocksPerRequestConfig struct { - Blocks int - Logs int - Traces int + Blocks int + Logs int + Traces int + Receipts int } type IRPCClient interface { @@ -44,13 +45,14 @@ type IRPCClient interface { } type Client struct { - RPCClient *gethRpc.Client - EthClient *ethclient.Client - supportsTraceBlock bool - isWebsocket bool - url string - chainID *big.Int - blocksPerRequest BlocksPerRequestConfig + RPCClient *gethRpc.Client + EthClient *ethclient.Client + supportsTraceBlock bool + supportsBlockReceipts bool + isWebsocket bool + url string + chainID *big.Int + blocksPerRequest BlocksPerRequestConfig } func Initialize() (IRPCClient, error) { @@ -111,28 +113,75 @@ func (rpc *Client) Close() { } func (rpc *Client) checkSupportedMethods() error { + if err := rpc.checkGetBlockByNumberSupport(); err != nil { + return err + } + if err := rpc.checkGetBlockReceiptsSupport(); err != nil { + return err + } + if err := rpc.checkGetLogsSupport(); err != nil { + return err + } + if err := rpc.checkTraceBlockSupport(); err != nil { + return err + } + return nil +} + +func (rpc *Client) checkGetBlockByNumberSupport() error { var blockByNumberResult interface{} err := rpc.RPCClient.Call(&blockByNumberResult, "eth_getBlockByNumber", "latest", true) if err != nil { return fmt.Errorf("eth_getBlockByNumber method not supported: %v", err) } log.Debug().Msg("eth_getBlockByNumber method supported") + return nil +} +func (rpc *Client) checkGetBlockReceiptsSupport() error { + if config.Cfg.RPC.BlockReceipts.Enabled { + var getBlockReceiptsResult interface{} + receiptsErr := rpc.RPCClient.Call(&getBlockReceiptsResult, "eth_getBlockReceipts", "latest") + if receiptsErr != nil { + log.Warn().Err(receiptsErr).Msg("eth_getBlockReceipts method not supported") + return fmt.Errorf("eth_getBlockReceipts method not supported: %v", receiptsErr) + } else { + rpc.supportsBlockReceipts = true + log.Debug().Msg("eth_getBlockReceipts method supported") + } + } else { + rpc.supportsBlockReceipts = false + log.Debug().Msg("eth_getBlockReceipts method disabled") + } + return nil +} + +func (rpc *Client) checkGetLogsSupport() error { + if rpc.supportsBlockReceipts { + return nil + } var getLogsResult interface{} logsErr := rpc.RPCClient.Call(&getLogsResult, "eth_getLogs", map[string]string{"fromBlock": "0x0", "toBlock": "0x0"}) if logsErr != nil { return fmt.Errorf("eth_getLogs method not supported: %v", logsErr) } log.Debug().Msg("eth_getLogs method supported") + return nil +} - var traceBlockResult interface{} +func (rpc *Client) checkTraceBlockSupport() error { if config.Cfg.RPC.Traces.Enabled { + var traceBlockResult interface{} if traceBlockErr := rpc.RPCClient.Call(&traceBlockResult, "trace_block", "latest"); traceBlockErr != nil { log.Warn().Err(traceBlockErr).Msg("Optional method trace_block not supported") + } else { + rpc.supportsTraceBlock = true + log.Debug().Msg("trace_block method supported") } + } else { + rpc.supportsTraceBlock = false + log.Debug().Msg("trace_block method disabled") } - rpc.supportsTraceBlock = traceBlockResult != nil - log.Debug().Msgf("trace_block method supported: %v", rpc.supportsTraceBlock) return nil } @@ -147,33 +196,44 @@ func (rpc *Client) setChainID() error { func (rpc *Client) GetFullBlocks(blockNumbers []*big.Int) []GetFullBlockResult { var wg sync.WaitGroup - var blocks []RPCFetchBatchResult[common.RawBlock] - var logs []RPCFetchBatchResult[common.RawLogs] - var traces []RPCFetchBatchResult[common.RawTraces] - + var blocks *[]RPCFetchBatchResult[common.RawBlock] + var logs *[]RPCFetchBatchResult[common.RawLogs] + var traces *[]RPCFetchBatchResult[common.RawTraces] + var receipts *[]RPCFetchBatchResult[common.RawReceipts] wg.Add(2) go func() { defer wg.Done() - blocks = RPCFetchBatch[common.RawBlock](rpc, blockNumbers, "eth_getBlockByNumber", GetBlockWithTransactionsParams) + result := RPCFetchBatch[common.RawBlock](rpc, blockNumbers, "eth_getBlockByNumber", GetBlockWithTransactionsParams) + blocks = &result }() - go func() { - defer wg.Done() - logs = RPCFetchInBatches[common.RawLogs](rpc, blockNumbers, rpc.blocksPerRequest.Logs, config.Cfg.RPC.Logs.BatchDelay, "eth_getLogs", GetLogsParams) - }() + if rpc.supportsBlockReceipts { + go func() { + defer wg.Done() + result := RPCFetchInBatches[common.RawReceipts](rpc, blockNumbers, rpc.blocksPerRequest.Receipts, config.Cfg.RPC.BlockReceipts.BatchDelay, "eth_getBlockReceipts", GetBlockReceiptsParams) + receipts = &result + }() + } else { + go func() { + defer wg.Done() + result := RPCFetchInBatches[common.RawLogs](rpc, blockNumbers, rpc.blocksPerRequest.Logs, config.Cfg.RPC.Logs.BatchDelay, "eth_getLogs", GetLogsParams) + logs = &result + }() + } if rpc.supportsTraceBlock { wg.Add(1) go func() { defer wg.Done() - traces = RPCFetchInBatches[common.RawTraces](rpc, blockNumbers, rpc.blocksPerRequest.Traces, config.Cfg.RPC.Traces.BatchDelay, "trace_block", TraceBlockParams) + result := RPCFetchInBatches[common.RawTraces](rpc, blockNumbers, rpc.blocksPerRequest.Traces, config.Cfg.RPC.Traces.BatchDelay, "trace_block", TraceBlockParams) + traces = &result }() } wg.Wait() - return SerializeFullBlocks(rpc.chainID, blocks, logs, traces) + return SerializeFullBlocks(rpc.chainID, blocks, logs, traces, receipts) } func (rpc *Client) GetBlocks(blockNumbers []*big.Int) []GetBlocksResult { diff --git a/internal/rpc/serializer.go b/internal/rpc/serializer.go index 36a488c..cac2e31 100644 --- a/internal/rpc/serializer.go +++ b/internal/rpc/serializer.go @@ -10,20 +10,17 @@ import ( "github.com/thirdweb-dev/indexer/internal/common" ) -func SerializeFullBlocks(chainId *big.Int, blocks []RPCFetchBatchResult[common.RawBlock], logs []RPCFetchBatchResult[common.RawLogs], traces []RPCFetchBatchResult[common.RawTraces]) []GetFullBlockResult { - results := make([]GetFullBlockResult, 0, len(blocks)) - - rawLogsMap := make(map[string]RPCFetchBatchResult[common.RawLogs]) - for _, rawLogs := range logs { - rawLogsMap[rawLogs.BlockNumber.String()] = rawLogs +func SerializeFullBlocks(chainId *big.Int, blocks *[]RPCFetchBatchResult[common.RawBlock], logs *[]RPCFetchBatchResult[common.RawLogs], traces *[]RPCFetchBatchResult[common.RawTraces], receipts *[]RPCFetchBatchResult[common.RawReceipts]) []GetFullBlockResult { + if blocks == nil { + return []GetFullBlockResult{} } + results := make([]GetFullBlockResult, 0, len(*blocks)) - rawTracesMap := make(map[string]RPCFetchBatchResult[common.RawTraces]) - for _, rawTraces := range traces { - rawTracesMap[rawTraces.BlockNumber.String()] = rawTraces - } + rawLogsMap := mapBatchResultsByBlockNumber[common.RawLogs](logs) + rawReceiptsMap := mapBatchResultsByBlockNumber[common.RawReceipts](receipts) + rawTracesMap := mapBatchResultsByBlockNumber[common.RawTraces](traces) - for _, rawBlock := range blocks { + for _, rawBlock := range *blocks { result := GetFullBlockResult{ BlockNumber: rawBlock.BlockNumber, } @@ -42,13 +39,22 @@ func SerializeFullBlocks(chainId *big.Int, blocks []RPCFetchBatchResult[common.R result.Data.Block = serializeBlock(chainId, rawBlock.Result) blockTimestamp := result.Data.Block.Timestamp - result.Data.Transactions = serializeTransactions(chainId, rawBlock.Result["transactions"].([]interface{}), blockTimestamp) - if rawLogs, exists := rawLogsMap[rawBlock.BlockNumber.String()]; exists { - if rawLogs.Error != nil { - result.Error = rawLogs.Error + if rawReceipts, exists := rawReceiptsMap[rawBlock.BlockNumber.String()]; exists { + if rawReceipts.Error != nil { + result.Error = rawReceipts.Error } else { - result.Data.Logs = serializeLogs(chainId, rawLogs.Result, result.Data.Block) + result.Data.Logs = serializeLogsFromReceipts(chainId, &rawReceipts.Result, result.Data.Block) + result.Data.Transactions = serializeTransactions(chainId, rawBlock.Result["transactions"].([]interface{}), blockTimestamp, &rawReceipts.Result) + } + } else { + if rawLogs, exists := rawLogsMap[rawBlock.BlockNumber.String()]; exists { + if rawLogs.Error != nil { + result.Error = rawLogs.Error + } else { + result.Data.Logs = serializeLogs(chainId, rawLogs.Result, result.Data.Block) + result.Data.Transactions = serializeTransactions(chainId, rawBlock.Result["transactions"].([]interface{}), blockTimestamp, nil) + } } } @@ -68,6 +74,17 @@ func SerializeFullBlocks(chainId *big.Int, blocks []RPCFetchBatchResult[common.R return results } +func mapBatchResultsByBlockNumber[T any](results *[]RPCFetchBatchResult[T]) map[string]*RPCFetchBatchResult[T] { + if results == nil { + return make(map[string]*RPCFetchBatchResult[T], 0) + } + resultsMap := make(map[string]*RPCFetchBatchResult[T], len(*results)) + for _, result := range *results { + resultsMap[result.BlockNumber.String()] = &result + } + return resultsMap +} + func SerializeBlocks(chainId *big.Int, blocks []RPCFetchBatchResult[common.RawBlock]) []GetBlocksResult { results := make([]GetBlocksResult, 0, len(blocks)) @@ -122,23 +139,32 @@ func serializeBlock(chainId *big.Int, block common.RawBlock) common.Block { } } -func serializeTransactions(chainId *big.Int, transactions []interface{}, blockTimestamp uint64) []common.Transaction { +func serializeTransactions(chainId *big.Int, transactions []interface{}, blockTimestamp uint64, receipts *common.RawReceipts) []common.Transaction { if len(transactions) == 0 { return []common.Transaction{} } + receiptMap := make(map[string]*common.RawReceipt) + if receipts != nil && len(*receipts) > 0 { + for _, receipt := range *receipts { + txHash := interfaceToString(receipt["transactionHash"]) + if txHash != "" { + receiptMap[txHash] = &receipt + } + } + } serializedTransactions := make([]common.Transaction, 0, len(transactions)) - for _, tx := range transactions { - serializedTransactions = append(serializedTransactions, serializeTransaction(chainId, tx, blockTimestamp)) + for _, rawTx := range transactions { + tx, ok := rawTx.(map[string]interface{}) + if !ok { + log.Debug().Msgf("Failed to serialize transaction: %v", rawTx) + continue + } + serializedTransactions = append(serializedTransactions, serializeTransaction(chainId, tx, blockTimestamp, receiptMap[interfaceToString(tx["hash"])])) } return serializedTransactions } -func serializeTransaction(chainId *big.Int, rawTx interface{}, blockTimestamp uint64) common.Transaction { - tx, ok := rawTx.(map[string]interface{}) - if !ok { - log.Debug().Msgf("Failed to serialize transaction: %v", rawTx) - return common.Transaction{} - } +func serializeTransaction(chainId *big.Int, tx map[string]interface{}, blockTimestamp uint64, receipt *common.RawReceipt) common.Transaction { return common.Transaction{ ChainId: chainId, Hash: interfaceToString(tx["hash"]), @@ -166,7 +192,78 @@ func serializeTransaction(chainId *big.Int, rawTx interface{}, blockTimestamp ui R: hexToBigInt(tx["r"]), S: hexToBigInt(tx["s"]), V: hexToBigInt(tx["v"]), - AccessListJson: interfaceToJsonString(tx["accessList"]), + AccessListJson: func() *string { + if tx["accessList"] != nil { + jsonString := interfaceToJsonString(tx["accessList"]) + if jsonString == "" { + return nil + } + return &jsonString + } + return nil + }(), + ContractAddress: func() *string { + if receipt != nil { + contractAddress := interfaceToString((*receipt)["contractAddress"]) + if contractAddress == "" { + return nil + } + return &contractAddress + } + return nil + }(), + GasUsed: func() *uint64 { + if receipt != nil { + gasUsed := hexToUint64((*receipt)["gasUsed"]) + return &gasUsed + } + return nil + }(), + CumulativeGasUsed: func() *uint64 { + if receipt != nil { + cumulativeGasUsed := hexToUint64((*receipt)["cumulativeGasUsed"]) + return &cumulativeGasUsed + } + return nil + }(), + EffectiveGasPrice: func() *big.Int { + if receipt != nil { + effectiveGasPrice := hexToBigInt((*receipt)["effectiveGasPrice"]) + return effectiveGasPrice + } + return nil + }(), + BlobGasUsed: func() *uint64 { + if receipt != nil { + blobGasUsed := hexToUint64((*receipt)["blobGasUsed"]) + return &blobGasUsed + } + return nil + }(), + BlobGasPrice: func() *big.Int { + if receipt != nil { + blobGasPrice := hexToBigInt((*receipt)["blobGasPrice"]) + return blobGasPrice + } + return nil + }(), + LogsBloom: func() *string { + if receipt != nil { + logsBloom := interfaceToString((*receipt)["logsBloom"]) + if logsBloom == "" { + return nil + } + return &logsBloom + } + return nil + }(), + Status: func() *uint64 { + if receipt != nil { + status := hexToUint64((*receipt)["status"]) + return &status + } + return nil + }(), } } @@ -180,6 +277,30 @@ func extractFunctionSelector(s string) string { return s[0:10] } +func serializeLogsFromReceipts(chainId *big.Int, rawReceipts *[]map[string]interface{}, block common.Block) []common.Log { + logs := make([]common.Log, 0) + if rawReceipts == nil { + return logs + } + + for _, receipt := range *rawReceipts { + rawLogs, ok := receipt["logs"].([]interface{}) + if !ok { + log.Debug().Msgf("Failed to serialize logs: %v", receipt["logs"]) + continue + } + for _, rawLog := range rawLogs { + logMap, ok := rawLog.(map[string]interface{}) + if !ok { + log.Debug().Msgf("Invalid log format: %v", rawLog) + continue + } + logs = append(logs, serializeLog(chainId, logMap, block)) + } + } + return logs +} + func serializeLogs(chainId *big.Int, rawLogs []map[string]interface{}, block common.Block) []common.Log { serializedLogs := make([]common.Log, len(rawLogs)) for i, rawLog := range rawLogs { diff --git a/internal/rpc/utils.go b/internal/rpc/utils.go index 26f2cd4..832f32d 100644 --- a/internal/rpc/utils.go +++ b/internal/rpc/utils.go @@ -6,9 +6,10 @@ import ( // TODO: we should detect this automatically const ( - DEFAULT_BLOCKS_PER_REQUEST = 1000 - DEFAULT_LOGS_PER_REQUEST = 100 - DEFAULT_TRACES_PER_REQUEST = 100 + DEFAULT_BLOCKS_PER_REQUEST = 1000 + DEFAULT_LOGS_PER_REQUEST = 100 + DEFAULT_TRACES_PER_REQUEST = 100 + DEFAULT_RECEIPTS_PER_REQUEST = 250 ) func GetBlockPerRequestConfig() BlocksPerRequestConfig { @@ -24,10 +25,14 @@ func GetBlockPerRequestConfig() BlocksPerRequestConfig { if tracesBlocksPerRequest == 0 { tracesBlocksPerRequest = DEFAULT_TRACES_PER_REQUEST } - + blockReceiptsBlocksPerRequest := config.Cfg.RPC.BlockReceipts.BlocksPerRequest + if blockReceiptsBlocksPerRequest == 0 { + blockReceiptsBlocksPerRequest = DEFAULT_RECEIPTS_PER_REQUEST + } return BlocksPerRequestConfig{ - Blocks: blocksPerRequest, - Logs: logsBlocksPerRequest, - Traces: tracesBlocksPerRequest, + Blocks: blocksPerRequest, + Logs: logsBlocksPerRequest, + Traces: tracesBlocksPerRequest, + Receipts: blockReceiptsBlocksPerRequest, } } diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index 8ecdb5a..e672da4 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -112,7 +112,7 @@ func (c *ClickHouseConnector) insertTransactions(txs *[]common.Transaction) erro INSERT INTO ` + c.cfg.Database + `.transactions ( chain_id, hash, nonce, block_hash, block_number, block_timestamp, transaction_index, from_address, to_address, value, gas, gas_price, data, function_selector, max_fee_per_gas, max_priority_fee_per_gas, - transaction_type, r, s, v, access_list + transaction_type, r, s, v, access_list, contract_address, gas_used, cumulative_gas_used, effective_gas_price, blob_gas_used, blob_gas_price, logs_bloom, status ) ` batch, err := c.conn.PrepareBatch(context.Background(), query) @@ -142,6 +142,14 @@ func (c *ClickHouseConnector) insertTransactions(txs *[]common.Transaction) erro tx.S, tx.V, tx.AccessListJson, + tx.ContractAddress, + tx.GasUsed, + tx.CumulativeGasUsed, + tx.EffectiveGasPrice, + tx.BlobGasUsed, + tx.BlobGasPrice, + tx.LogsBloom, + tx.Status, ) if err != nil { return err diff --git a/internal/tools/clickhouse_create_transactions_table.sql b/internal/tools/clickhouse_create_transactions_table.sql index f43c72e..7487316 100644 --- a/internal/tools/clickhouse_create_transactions_table.sql +++ b/internal/tools/clickhouse_create_transactions_table.sql @@ -19,7 +19,15 @@ CREATE TABLE transactions ( `r` UInt256, `s` UInt256, `v` UInt256, - `access_list` Nullable(String), + `access_list` Nullable(String), + `contract_address` Nullable(FixedString(42)), + `gas_used` Nullable(UInt64), + `cumulative_gas_used` Nullable(UInt64), + `effective_gas_price` Nullable(UInt256), + `blob_gas_used` Nullable(UInt64), + `blob_gas_price` Nullable(UInt256), + `logs_bloom` Nullable(String), + `status` Nullable(UInt64), `is_deleted` UInt8 DEFAULT 0, `insert_timestamp` DateTime DEFAULT now(), INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,