From 2bca620338dfc313bbaa048819904279e63d5ba4 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 28 Aug 2024 16:55:24 +0300 Subject: [PATCH 01/17] Add historical synchronization. --- blockchain/arbitrum_one/arbitrum_one.go | 23 ++ .../arbitrum_sepolia/arbitrum_sepolia.go | 23 ++ blockchain/blockchain.go.tmpl | 23 ++ blockchain/ethereum/ethereum.go | 23 ++ .../game7_orbit_arbitrum_sepolia.go | 23 ++ blockchain/game7_testnet/game7_testnet.go | 23 ++ blockchain/handlers.go | 124 ++++++ blockchain/imx_zkevm/imx_zkevm.go | 23 ++ .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 23 ++ blockchain/mantle/mantle.go | 23 ++ blockchain/mantle_sepolia/mantle_sepolia.go | 23 ++ blockchain/polygon/polygon.go | 23 ++ blockchain/sepolia/sepolia.go | 23 ++ blockchain/xai/xai.go | 23 ++ blockchain/xai_sepolia/xai_sepolia.go | 23 ++ cmd.go | 97 ++++- indexer/db.go | 360 +++++++++++++++++- indexer/types.go | 8 +- synchronizer/synchronizer.go | 235 +++++++++++- 19 files changed, 1115 insertions(+), 31 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index a867fc7..13c5774 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index 868ca3d..55c050c 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 4401d37..99eecf8 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index ab0676e..79c23af 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 680ff22..95e8162 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 90ccdaa..ab77ca1 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/handlers.go b/blockchain/handlers.go index 0d1680f..f5bad15 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -2,13 +2,16 @@ package blockchain import ( "bytes" + "context" "errors" "fmt" "log" "math/big" + "sync" "time" "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" "github.com/moonstream-to/seer/blockchain/arbitrum_one" "github.com/moonstream-to/seer/blockchain/arbitrum_sepolia" seer_common "github.com/moonstream-to/seer/blockchain/common" @@ -88,6 +91,7 @@ type BlockchainClient interface { DecodeProtoEntireBlockToLabels(*bytes.Buffer, map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) ChainType() string + GetCode(context.Context, common.Address, uint64) ([]byte, error) } func GetLatestBlockNumberWithRetry(client BlockchainClient, retryAttempts int, retryWaitTime time.Duration) (*big.Int, error) { @@ -136,3 +140,123 @@ func DecodeTransactionInputData(contractABI *abi.ABI, data []byte) { fmt.Printf("Method Name: %s\n", method.Name) fmt.Printf("Method inputs: %v\n", inputsMap) } + +func DeployBlocksLookUpAndUpdate() error { + + // get all abi jobs without deployed block + + chainsAddresses, err := indexer.DBConnection.GetAbiJobsWithoutDeployBlocks() + + if err != nil { + log.Printf("Failed to get abi jobs without deployed blocks: %v", err) + return err + } + + if len(chainsAddresses) == 0 { + log.Printf("No abi jobs without deployed blocks") + return nil + } + + for chain, addresses := range chainsAddresses { + + var wg sync.WaitGroup + + sem := make(chan struct{}, 5) // Semaphore to control + errChan := make(chan error, 1) // Buffered channel for error handling + + for address, ids := range addresses { + + wg.Add(1) + go func(address string, chain string, ids []string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + client, err := NewClient(chain, "", 0) + + if err != nil { + errChan <- err + return + } + + // get all abi jobs without deployed block + deployedBlock, err := FindDeployedBlock(client, address) + + if err != nil { + errChan <- err + return + } + + if deployedBlock != 0 { + // update abi job with deployed block + err := indexer.DBConnection.UpdateAbiJobsDeployBlock(deployedBlock, ids) + if err != nil { + errChan <- err + return + } + } + + }(address, chain, ids) + + } + + go func() { + wg.Wait() + close(errChan) + }() + + for err := range errChan { + if err != nil { + log.Printf("Failed to get deployed block: %v", err) + return err + } + } + + } + + return nil + +} + +func FindDeployedBlock(client BlockchainClient, address string) (uint64, error) { + + // Binary search by get code + + ctx := context.Background() + + // with timeout + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + + defer cancel() + + latestBlockNumber, err := client.GetLatestBlockNumber() + + if err != nil { + return 0, err + } + + var left uint64 = 0 + + var right uint64 = latestBlockNumber.Uint64() + + for left < right { + + mid := (left + right) / 2 + + code, err := client.GetCode(ctx, common.HexToAddress(address), mid) + + if err != nil { + return 0, err + } + + if code == nil { + right = mid + } else { + left = mid + 1 + } + + } + + return left, nil +} diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index f6f0df6..878e5e4 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index d99fab5..be4dae5 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 0f44560..fd709fc 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index a140ca5..b0397b9 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index ff5671e..9cbe480 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index 89b9f43..db02b1f 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 85cbbf2..f7a2f75 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 98d475b..7052117 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -105,6 +105,29 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code string + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + if err != nil { + return nil, err + } + + if code == "0x" { + return nil, nil + } + + return common.FromHex(code), nil +} + func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/cmd.go b/cmd.go index 998153a..57db14d 100644 --- a/cmd.go +++ b/cmd.go @@ -46,7 +46,8 @@ func CreateRootCommand() *cobra.Command { synchronizerCmd := CreateSynchronizerCommand() abiCmd := CreateAbiCommand() dbCmd := CreateDatabaseOperationCommand() - rootCmd.AddCommand(completionCmd, versionCmd, blockchainCmd, starknetCmd, evmCmd, crawlerCmd, inspectorCmd, synchronizerCmd, abiCmd, dbCmd) + historicalSyncCmd := CreateHistoricalSyncCommand() + rootCmd.AddCommand(completionCmd, versionCmd, blockchainCmd, starknetCmd, evmCmd, crawlerCmd, inspectorCmd, synchronizerCmd, abiCmd, dbCmd, historicalSyncCmd) // By default, cobra Command objects write to stderr. We have to forcibly set them to output to // stdout. @@ -760,11 +761,105 @@ func CreateDatabaseOperationCommand() *cobra.Command { indexCommand.AddCommand(cleanCommand) + deploymentBlocksCommand := &cobra.Command{ + Use: "deployment-blocks", + Short: "Get deployment blocks from address in abi jobs", + PreRunE: func(cmd *cobra.Command, args []string) error { + indexerErr := indexer.CheckVariablesForIndexer() + if indexerErr != nil { + return indexerErr + } + + indexer.InitDBConnection() + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + + deploymentBlocksErr := seer_blockchain.DeployBlocksLookUpAndUpdate() + if deploymentBlocksErr != nil { + return deploymentBlocksErr + } + + return nil + }, + } + + databaseCmd.AddCommand(deploymentBlocksCommand) databaseCmd.AddCommand(indexCommand) return databaseCmd } +func CreateHistoricalSyncCommand() *cobra.Command { + + var chain, baseDir, customerDbUriFlag string + var addresses, customerIds []string + var startBlock, endBlock, batchSize uint64 + var timeout int + var auto bool + + historicalSyncCmd := &cobra.Command{ + Use: "historical-sync", + Short: "Decode the historical data from various blockchains", + PreRunE: func(cmd *cobra.Command, args []string) error { + indexerErr := indexer.CheckVariablesForIndexer() + if indexerErr != nil { + return indexerErr + } + + storageErr := storage.CheckVariablesForStorage() + if storageErr != nil { + return storageErr + } + + crawlerErr := crawler.CheckVariablesForCrawler() + if crawlerErr != nil { + return crawlerErr + } + + syncErr := synchronizer.CheckVariablesForSynchronizer() + if syncErr != nil { + return syncErr + } + + if chain == "" { + return fmt.Errorf("blockchain is required via --chain") + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + indexer.InitDBConnection() + + newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout) + if synchonizerErr != nil { + return synchonizerErr + } + + err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, customerIds, addresses, batchSize, auto) + + if err != nil { + return err + } + + return nil + }, + } + + historicalSyncCmd.Flags().StringVar(&chain, "chain", "ethereum", "The blockchain to crawl (default: ethereum)") + historicalSyncCmd.Flags().StringVar(&baseDir, "base-dir", "", "The base directory to store the crawled data (default: '')") + historicalSyncCmd.Flags().Uint64Var(&startBlock, "start-block", 0, "The block number to start decoding from (default: latest block)") + historicalSyncCmd.Flags().Uint64Var(&endBlock, "end-block", 0, "The block number to end decoding at (default: latest block)") + historicalSyncCmd.Flags().IntVar(&timeout, "timeout", 30, "The timeout for the crawler in seconds (default: 30)") + historicalSyncCmd.Flags().Uint64Var(&batchSize, "batch-size", 100, "The number of blocks to crawl in each batch (default: 100)") + historicalSyncCmd.Flags().StringVar(&customerDbUriFlag, "customer-db-uri", "", "Set customer database URI for development. This workflow bypass fetching customer IDs and its database URL connection strings from mdb-v3-controller API") + historicalSyncCmd.Flags().StringSliceVar(&customerIds, "customer-ids", []string{}, "The list of customer IDs to sync") + historicalSyncCmd.Flags().StringSliceVar(&addresses, "addresses", []string{}, "The list of addresses to sync") + historicalSyncCmd.Flags().BoolVar(&auto, "auto", false, "Set this flag to sync all unfinished historical crawl from the database (default: false)") + + return historicalSyncCmd +} + func CreateStarknetParseCommand() *cobra.Command { var infile string var rawABI []byte diff --git a/indexer/db.go b/indexer/db.go index 03041ef..ca19cc4 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -460,7 +460,7 @@ func (p *PostgreSQLpgx) ReadABIJobs(blockchain string) ([]AbiJob, error) { defer conn.Release() - rows, err := conn.Query(context.Background(), "SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, (abi::jsonb)->>'type' as abiType, created_at, updated_at FROM abi_jobs where chain=$1 and (abi::jsonb)->>'type' is not null", blockchain) + rows, err := conn.Query(context.Background(), "SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, (abi::jsonb)->>'type' as abiType, created_at, updated_at, deployment_block_number FROM abi_jobs where chain=$1 and (abi::jsonb)->>'type' is not null", blockchain) if err != nil { return nil, err @@ -519,14 +519,14 @@ func (p *PostgreSQLpgx) GetCustomersIDs(blockchain string) ([]string, error) { return customerIds, nil } -func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, customerIds []string) (uint64, string, []CustomerUpdates, error) { +func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, customerIds []string) (uint64, uint64, string, []CustomerUpdates, error) { pool := p.GetPool() conn, err := pool.Acquire(context.Background()) if err != nil { - return 0, "", nil, err + return 0, 0, "", nil, err } defer conn.Release() @@ -607,18 +607,18 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome if err != nil { log.Println("Error querying abi jobs from database", err) - return 0, "", nil, err + return 0, 0, "", nil, err } var customers []map[string]map[string]map[string]map[string]string var path string - var lastNumber uint64 + var firstBlockNumber, lastBlockNumber uint64 for rows.Next() { - err = rows.Scan(&lastNumber, &path, &customers) + err = rows.Scan(&lastBlockNumber, &path, &customers) if err != nil { log.Println("Error scanning row:", err) - return 0, "", nil, err + return 0, 0, "", nil, err } } @@ -637,7 +637,7 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome } - return lastNumber, path, customerUpdates, nil + return firstBlockNumber, lastBlockNumber, path, customerUpdates, nil } @@ -1065,3 +1065,347 @@ func (p *PostgreSQLpgx) CleanIndexes(blockchain string, batchLimit uint64, sleep return nil } + +func (p *PostgreSQLpgx) UpdateAbiJobsStatus(blockchain string) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return err + } + defer conn.Release() + + query := ` + UPDATE abi_jobs + SET historical_crawl_status = 'pickedup' + WHERE chain = @chain + AND historical_crawl_status = 'pending' + AND status = true + AND deployment_block_number IS NOT NULL + ` + + queryArgs := pgx.NamedArgs{ + "chain": blockchain, + } + + _, err = conn.Exec(context.Background(), query, queryArgs) + if err != nil { + return err + } + + return nil +} + +func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, customersIds []string) ([]CustomerUpdates, map[string]AbiJobsDeployInfo, error) { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return nil, nil, err + } + defer conn.Release() + + var queryBuilder strings.Builder + + queryArgs := make(pgx.NamedArgs) + + queryArgs["chain"] = blockchain + + queryBuilder.WriteString(` + SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, + historical_crawl_status, progress, task_pickedup, abi, + (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number + FROM abi_jobs + WHERE chain = @chain AND historical_crawl_status = 'pickedup' + `) + + if len(addresses) > 0 { + queryBuilder.WriteString(" AND address = ANY(@addresses) ") + queryArgs["addresses"] = addresses + } + + if len(customersIds) > 0 { + queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") + queryArgs["customer_ids"] = customersIds + } + + rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) + if err != nil { + return nil, nil, err + } + + abiJobs, err := pgx.CollectRows(rows, pgx.RowToStructByName[AbiJob]) + if err != nil { + log.Println("Error collecting ABI jobs rows", err) + return nil, nil, err + } + + if len(abiJobs) == 0 { + return []CustomerUpdates{}, map[string]AbiJobsDeployInfo{}, nil + } + + customerUpdatesDict := make(map[string]CustomerUpdates) + addressDeployBlockDict := make(map[string]AbiJobsDeployInfo) + + for _, abiJob := range abiJobs { + address := fmt.Sprintf("0x%x", abiJob.Address) + + if _, exists := customerUpdatesDict[abiJob.CustomerID]; !exists { + customerUpdatesDict[abiJob.CustomerID] = CustomerUpdates{ + CustomerID: abiJob.CustomerID, + Abis: make(map[string]map[string]map[string]string), + } + } + + if _, exists := customerUpdatesDict[abiJob.CustomerID].Abis[address]; !exists { + customerUpdatesDict[abiJob.CustomerID].Abis[address] = make(map[string]map[string]string) + } + + customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = map[string]string{ + "abi": abiJob.Abi, + "abi_name": abiJob.AbiName, + } + + // Retrieve the struct from the map + deployInfo, exists := addressDeployBlockDict[address] + if !exists { + // Initialize the struct if it doesn't exist + deployInfo = AbiJobsDeployInfo{ + DeployedBlockNumber: abiJob.DeploymentBlockNumber, + IDs: []string{}, + } + } + + // Modify the struct + deployInfo.IDs = append(deployInfo.IDs, abiJob.ID) + + // Store the modified struct back in the map + addressDeployBlockDict[address] = deployInfo + } + + var customerUpdates []CustomerUpdates + for _, customerUpdate := range customerUpdatesDict { + customerUpdates = append(customerUpdates, customerUpdate) + } + + return customerUpdates, addressDeployBlockDict, nil +} + +func (p *PostgreSQLpgx) UpdateAbisAsDone(ids []string) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return err + } + defer conn.Release() + + query := ` + UPDATE abi_jobs + SET historical_crawl_status = 'done' + WHERE id = ANY($1) + ` + + _, err = conn.Exec(context.Background(), query, ids) + if err != nil { + return err + } + + return nil +} + +// func (p *PostgreSQLpgx) SelectAndUpdateUnprocessedAbi(blockchain string, addresses []string, customersIds []string) ([]CustomerUpdates, map[string]uint64, error) { +// pool := p.GetPool() + +// conn, err := pool.Acquire(context.Background()) +// if err != nil { +// return nil, nil, err +// } +// defer conn.Release() + +// var queryBuilder strings.Builder + +// queryArgs := make(pgx.NamedArgs) + +// queryArgs["chain"] = blockchain + +// if len(addresses) > 0 { +// queryBuilder.WriteString(` +// UPDATE abi_jobs +// SET historical_crawl_status = 'pickedup' +// WHERE chain = @chain +// AND address = ANY(@addresses) +// `) +// queryArgs["addresses"] = addresses +// } else { +// queryBuilder.WriteString(` +// UPDATE abi_jobs +// SET historical_crawl_status = 'pickedup' +// WHERE chain = @chain +// AND historical_crawl_status = 'pending' +// AND status = true +// AND deployment_block_number IS NOT NULL +// `) +// } + +// if len(customersIds) > 0 { + +// queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") +// queryArgs["customer_ids"] = customersIds +// } + +// queryBuilder.WriteString(` +// RETURNING id, address, user_id, customer_id, abi_selector, chain, abi_name, status, +// historical_crawl_status, progress, moonworm_task_pickedup, abi, +// (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number +// `) + +// rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) +// if err != nil { +// return nil, nil, err +// } + +// abiJobs, err := pgx.CollectRows(rows, pgx.RowToStructByName[AbiJob]) +// if err != nil { +// log.Println("Error collecting ABI jobs rows", err) +// return nil, nil, err +// } + +// if len(abiJobs) == 0 { +// return []CustomerUpdates{}, map[string]uint64{}, nil +// } + +// customerUpdatesDict := make(map[string]CustomerUpdates) +// addressDeployBlockDict := make(map[string]uint64) + +// for _, abiJob := range abiJobs { +// address := fmt.Sprintf("0x%x", abiJob.Address) + +// if _, exists := customerUpdatesDict[abiJob.CustomerID]; !exists { +// customerUpdatesDict[abiJob.CustomerID] = CustomerUpdates{ +// CustomerID: abiJob.CustomerID, +// Abis: make(map[string]map[string]map[string]string), +// } +// } + +// if _, exists := customerUpdatesDict[abiJob.CustomerID].Abis[address]; !exists { +// customerUpdatesDict[abiJob.CustomerID].Abis[address] = make(map[string]map[string]string) +// } + +// customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = map[string]string{ +// "abi": abiJob.Abi, +// "abi_name": abiJob.AbiName, +// } + +// addressDeployBlockDict[address] = abiJob.DeployedBlockNumber +// } + +// var customerUpdates []CustomerUpdates +// for _, customerUpdate := range customerUpdatesDict { +// customerUpdates = append(customerUpdates, customerUpdate) +// } + +// return customerUpdates, addressDeployBlockDict, nil +// } + +func (p *PostgreSQLpgx) FindBatchPath(blockchain string, blockNumber uint64) (string, uint64, uint64, error) { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return "", 0, 0, err + } + + defer conn.Release() + + var path string + + var minBlockNumber uint64 + + var maxBlockNumber uint64 + + query := fmt.Sprintf("SELECT path, min(block_number), max(block_number) FROM %s WHERE block_number = $1", BlocksTableName(blockchain)) + + // Check if we have at least one job before accessing + + err = conn.QueryRow(context.Background(), query, blockNumber).Scan(&path, &minBlockNumber, &maxBlockNumber) + + if err != nil { + return "", + 0, + 0, + err + } + + return path, minBlockNumber, maxBlockNumber, nil + +} + +func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks() (map[string]map[string][]string, error) { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return nil, err + } + + defer conn.Release() + + /// get all addresses that not have deploy block number + + rows, err := conn.Query(context.Background(), "SELECT id, chain, address FROM abi_jobs WHERE chain=$1 and deploy_block_number is null") + + if err != nil { + return nil, err + } + + // chain, address, ids + var chainsAddresses map[string]map[string][]string + + for rows.Next() { + + var id string + var chain string + var address string + + err = rows.Scan(&id, &chain, &address) + + if err != nil { + return nil, err + } + + if chainsAddresses[chain] == nil { + chainsAddresses[chain] = make(map[string][]string) + } + + chainsAddresses[chain][address] = append(chainsAddresses[chain][address], id) + + } + + return chainsAddresses, nil +} + +func (p *PostgreSQLpgx) UpdateAbiJobsDeployBlock(blockNumber uint64, ids []string) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return err + } + + defer conn.Release() + + // for id, blockNumber := range ids { + + _, err = conn.Exec(context.Background(), "UPDATE abi_jobs SET deploy_block_number=$1 WHERE id=$2", blockNumber, ids) + + if err != nil { + return err + } + + return nil + +} diff --git a/indexer/types.go b/indexer/types.go index 8438d2d..3c21048 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -60,11 +60,12 @@ type AbiJob struct { Status string HistoricalCrawlStatus string Progress int - MoonwormTaskPickedup bool + TaskPickedup bool Abi string AbiType string CreatedAt time.Time UpdatedAt time.Time + DeploymentBlockNumber uint64 } type CustomerUpdates struct { @@ -125,3 +126,8 @@ type TransactionLabel struct { LabelData string BlockTimestamp uint64 } + +type AbiJobsDeployInfo struct { + DeployedBlockNumber uint64 + IDs []string +} diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 953a5c2..dff69c7 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "math/big" "net/http" "net/url" "path/filepath" @@ -18,6 +19,7 @@ import ( "github.com/moonstream-to/seer/indexer" "github.com/moonstream-to/seer/storage" "golang.org/x/exp/slices" + "google.golang.org/protobuf/proto" ) type Synchronizer struct { @@ -155,30 +157,38 @@ type CustomerDBConnection struct { Pgx *indexer.PostgreSQLpgx } -// getCustomers fetch ABI jobs, customer IDs and database URLs -func (d *Synchronizer) getCustomers(customerDbUriFlag string) (map[string]CustomerDBConnection, []string, error) { +// getCustomers fetches ABI jobs, extracts customer IDs, and establishes database connections. +func (d *Synchronizer) getCustomers(customerDbUriFlag string, customerIds []string) (map[string]CustomerDBConnection, []string, error) { customerDBConnections := make(map[string]CustomerDBConnection) - var customerIds []string + customerIdSet := make(map[string]struct{}) - // Read ABI jobs from database - abiJobs, err := d.ReadAbiJobsFromDatabase(d.blockchain) - if err != nil { - return customerDBConnections, customerIds, err - } + // If no customer IDs are provided as arguments, fetch them from ABI jobs. + if len(customerIds) == 0 { + abiJobs, err := d.ReadAbiJobsFromDatabase(d.blockchain) + if err != nil { + return nil, nil, fmt.Errorf("failed to read ABI jobs: %w", err) + } - // Create a set of customer IDs from ABI jobs to remove duplicates - customerIdsSet := make(map[string]struct{}) - for _, job := range abiJobs { - customerIdsSet[job.CustomerID] = struct{}{} + // Extract unique customer IDs from the ABI jobs. + for _, job := range abiJobs { + customerIdSet[job.CustomerID] = struct{}{} + } + } else { + // Otherwise, use the provided customer IDs directly. + for _, id := range customerIds { + customerIdSet[id] = struct{}{} + } } - for id := range customerIdsSet { + var finalCustomerIds []string + for id := range customerIdSet { var connectionString string var dbConnErr error + if customerDbUriFlag == "" { connectionString, dbConnErr = GetDBConnection(id) if dbConnErr != nil { - log.Printf("Unable to get connection database URI for %s customer, err: %v", id, dbConnErr) + log.Printf("Unable to get connection database URI for customer %s, err: %v", id, dbConnErr) continue } } else { @@ -187,7 +197,7 @@ func (d *Synchronizer) getCustomers(customerDbUriFlag string) (map[string]Custom pgx, pgxErr := indexer.NewPostgreSQLpgxWithCustomURI(connectionString) if pgxErr != nil { - log.Printf("Error creating RDS connection for %s customer, err: %v", id, pgxErr) + log.Printf("Error creating RDS connection for customer %s, err: %v", id, pgxErr) continue } @@ -195,12 +205,12 @@ func (d *Synchronizer) getCustomers(customerDbUriFlag string) (map[string]Custom Uri: connectionString, Pgx: pgx, } - customerIds = append(customerIds, id) - + finalCustomerIds = append(finalCustomerIds, id) } - log.Println("Customer IDs to sync:", customerIds) - return customerDBConnections, customerIds, nil + log.Println("Customer IDs to sync:", finalCustomerIds) + + return customerDBConnections, finalCustomerIds, nil } func (d *Synchronizer) Start(customerDbUriFlag string) { @@ -234,7 +244,7 @@ func (d *Synchronizer) Start(customerDbUriFlag string) { func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { var isEnd bool - customerDBConnections, customerIds, customersErr := d.getCustomers(customerDbUriFlag) + customerDBConnections, customerIds, customersErr := d.getCustomers(customerDbUriFlag, nil) if customersErr != nil { return isEnd, customersErr } @@ -314,7 +324,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { // Read updates from the indexer db // This function will return a list of customer updates 1 update is 1 customer - lastBlockOfChank, path, updates, err := indexer.DBConnection.ReadUpdates(d.blockchain, d.startBlock, customerIds) + _, lastBlockOfChank, path, updates, err := indexer.DBConnection.ReadUpdates(d.blockchain, d.startBlock, customerIds) if len(updates) == 0 { log.Printf("No updates found for block %d\n", d.startBlock) @@ -405,3 +415,186 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, nil } + +func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool) error { + var useRPC bool + var isCycleFinished bool + + fmt.Println("HistoricalSyncRef") + + // Initialize start block if 0 + if d.startBlock == 0 { + // Get the latest block from the indexer db + indexedLatestBlock, err := indexer.DBConnection.GetLatestDBBlockNumber(d.blockchain) + if err != nil { + return fmt.Errorf("error getting latest block number: %w", err) + } + d.startBlock = indexedLatestBlock + } + + // Automatically update ABI jobs if auto mode is enabled + if auto { + if err := indexer.DBConnection.UpdateAbiJobsStatus(d.blockchain); err != nil { + return fmt.Errorf("error updating ABI: %w", err) + } + } + + // Retrieve customer updates and deployment blocks + customerUpdates, addressesAbisInfo, err := indexer.DBConnection.SelectAbiJobs(d.blockchain, addresses, customerIds) + if err != nil { + return fmt.Errorf("error selecting ABI jobs: %w", err) + } + + // Filter out blocks more + for address, abisInfo := range addressesAbisInfo { + if abisInfo.DeployedBlockNumber > d.startBlock { + delete(addressesAbisInfo, address) + } + } + + // Get customer database connections + customerIdsMap := make(map[string]bool) + for _, update := range customerUpdates { + customerIdsMap[update.CustomerID] = true + } + var customerIdsList []string + for id := range customerIdsMap { + customerIdsList = append(customerIdsList, id) + } + + customerDBConnections, _, err := d.getCustomers(customerDbUriFlag, customerIdsList) + if err != nil { + return fmt.Errorf("error getting customers: %w", err) + } + + // Main processing loop + for { + if d.endBlock != 0 && d.startBlock <= d.endBlock { + isCycleFinished = true + log.Printf("End block %d almost reached", d.endBlock) + } + + for address, abisInfo := range addressesAbisInfo { + if abisInfo.DeployedBlockNumber > d.startBlock { + + // update the status of the address for the customer to done + err := indexer.DBConnection.UpdateAbisAsDone(abisInfo.IDs) + if err != nil { + return err + } + + // drop the address + delete(addressesAbisInfo, address) + } + } + + if len(addressesAbisInfo) == 0 { + break + } + + // Determine the processing strategy (RPC or storage) + var path string + var firstBlockOfChunk uint64 + if !useRPC { + + path, firstBlockOfChunk, _, err = indexer.DBConnection.FindBatchPath(d.blockchain, d.startBlock) + if err != nil { + return fmt.Errorf("error finding batch path: %w", err) + } + + if path == "" { + useRPC = true + d.endBlock = d.startBlock - batchSize + } else { + d.endBlock = firstBlockOfChunk + } + } else { + d.endBlock = d.startBlock - batchSize + } + + // Read raw data from storage or via RPC + var rawData bytes.Buffer + if useRPC { + protoMessage, _, _, err := seer_blockchain.CrawlEntireBlocks(d.Client, big.NewInt(int64(d.startBlock)), big.NewInt(int64(firstBlockOfChunk)), false, 5) + if err != nil { + return fmt.Errorf("error reading events via RPC: %w", err) + } + + blocksBatch, err := d.Client.ProcessBlocksToBatch(protoMessage) + if err != nil { + return fmt.Errorf("error processing blocks to batch: %w", err) + } + + dataBytes, err := proto.Marshal(blocksBatch) + if err != nil { + return fmt.Errorf("error marshaling protoMessage: %w", err) + } + + rawData = *bytes.NewBuffer(dataBytes) + } else { + rawData, err = d.StorageInstance.Read(path) + if err != nil { + return fmt.Errorf("error reading events from storage: %w", err) + } + } + + log.Printf("Processing %d customer updates for block range %d-%d", len(customerUpdates), d.startBlock, firstBlockOfChunk) + + // Process customer updates in parallel + var wg sync.WaitGroup + sem := make(chan struct{}, 5) // Semaphore to control concurrency + errChan := make(chan error, 1) // Buffered channel for error handling + + for _, update := range customerUpdates { + wg.Add(1) + go func(update indexer.CustomerUpdates) { + defer wg.Done() + sem <- struct{}{} // Acquire semaphore + + customer, exists := customerDBConnections[update.CustomerID] + if !exists { + errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) + <-sem // Release semaphore + return + } + + conn, err := customer.Pgx.GetPool().Acquire(context.Background()) + if err != nil { + errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + defer conn.Release() + + decodedEvents, decodedTransactions, err := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis) + if err != nil { + errChan <- fmt.Errorf("error decoding events for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + + customer.Pgx.WriteLabes(d.blockchain, decodedTransactions, decodedEvents) + + <-sem // Release semaphore + }(update) + } + + wg.Wait() + close(sem) + + // Check for errors from goroutines + select { + case err := <-errChan: + return err + default: + } + + d.startBlock = d.endBlock - 1 + + if isCycleFinished { + break + } + } + + return nil +} From e80ffbc8d4a335f137db0583d5b8ec4e2fae6eb2 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 29 Aug 2024 13:29:30 +0300 Subject: [PATCH 02/17] Fix deploy blocks update. --- blockchain/arbitrum_one/arbitrum_one.go | 13 +- .../arbitrum_sepolia/arbitrum_sepolia.go | 13 +- blockchain/blockchain.go.tmpl | 13 +- blockchain/ethereum/ethereum.go | 13 +- .../game7_orbit_arbitrum_sepolia.go | 13 +- blockchain/game7_testnet/game7_testnet.go | 13 +- blockchain/handlers.go | 20 +- blockchain/imx_zkevm/imx_zkevm.go | 13 +- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 13 +- blockchain/mantle/mantle.go | 13 +- blockchain/mantle_sepolia/mantle_sepolia.go | 13 +- blockchain/polygon/polygon.go | 13 +- blockchain/sepolia/sepolia.go | 13 +- blockchain/xai/xai.go | 13 +- blockchain/xai_sepolia/xai_sepolia.go | 13 +- cmd.go | 10 +- crawler/crawler.go | 2 +- indexer/db.go | 198 +++++++----------- indexer/types.go | 2 +- 19 files changed, 186 insertions(+), 228 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 13c5774..95cccef 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index 55c050c..a92107f 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 99eecf8..94cd1c0 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 79c23af..6d50b50 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 95e8162..37c2993 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index ab77ca1..ab1cd90 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/handlers.go b/blockchain/handlers.go index f5bad15..13e6eab 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -164,6 +164,8 @@ func DeployBlocksLookUpAndUpdate() error { sem := make(chan struct{}, 5) // Semaphore to control errChan := make(chan error, 1) // Buffered channel for error handling + log.Printf("Processing chain: %s with amount of addresses: %d\n", chain, len(addresses)) + for address, ids := range addresses { wg.Add(1) @@ -172,7 +174,7 @@ func DeployBlocksLookUpAndUpdate() error { sem <- struct{}{} defer func() { <-sem }() - client, err := NewClient(chain, "", 0) + client, err := NewClient(chain, BlockchainURLs[chain], 4) if err != nil { errChan <- err @@ -187,6 +189,8 @@ func DeployBlocksLookUpAndUpdate() error { return } + log.Printf("Deployed block: %d for address: %s in chain: %s\n", deployedBlock, address, chain) + if deployedBlock != 0 { // update abi job with deployed block err := indexer.DBConnection.UpdateAbiJobsDeployBlock(deployedBlock, ids) @@ -236,24 +240,24 @@ func FindDeployedBlock(client BlockchainClient, address string) (uint64, error) return 0, err } - var left uint64 = 0 - + var left uint64 = 1 var right uint64 = latestBlockNumber.Uint64() + var code []byte for left < right { - mid := (left + right) / 2 - code, err := client.GetCode(ctx, common.HexToAddress(address), mid) + code, err = client.GetCode(ctx, common.HexToAddress(address), mid) if err != nil { + log.Printf("Failed to get code: %v", err) return 0, err } - if code == nil { - right = mid - } else { + if len(code) == 0 { left = mid + 1 + } else { + right = mid } } diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 878e5e4..06464b8 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index be4dae5..408619b 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index fd709fc..cc4cb71 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index b0397b9..53c291b 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 9cbe480..0bb98ad 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index db02b1f..bb12b29 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index f7a2f75..ff88b5e 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 7052117..7387848 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -107,7 +108,7 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ // Get bytecode of a contract by address. func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { - var code string + var code hexutil.Bytes if blockNumber == 0 { latestBlockNumber, err := c.GetLatestBlockNumber() if err != nil { @@ -115,19 +116,17 @@ func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumbe } blockNumber = latestBlockNumber.Uint64() } - - err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, blockNumber) + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) return nil, err } - if code == "0x" { + if len(code) == 0 { return nil, nil } - - return common.FromHex(code), nil + return code, nil } - func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock diff --git a/cmd.go b/cmd.go index 57db14d..c3ce596 100644 --- a/cmd.go +++ b/cmd.go @@ -697,7 +697,7 @@ func CreateAbiEnsureSelectorsCommand() *cobra.Command { indexer.InitDBConnection() - updateErr := indexer.DBConnection.EnsureCorrectSelectors(chain, WriteToDB, outFilePath) + updateErr := indexer.DBConnection.EnsureCorrectSelectors(chain, WriteToDB, outFilePath, []string{}) if updateErr != nil { return updateErr } @@ -769,8 +769,12 @@ func CreateDatabaseOperationCommand() *cobra.Command { if indexerErr != nil { return indexerErr } - + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } indexer.InitDBConnection() + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -784,7 +788,7 @@ func CreateDatabaseOperationCommand() *cobra.Command { }, } - databaseCmd.AddCommand(deploymentBlocksCommand) + indexCommand.AddCommand(deploymentBlocksCommand) databaseCmd.AddCommand(indexCommand) return databaseCmd diff --git a/crawler/crawler.go b/crawler/crawler.go index 9d0bfbd..b0b9271 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -106,7 +106,7 @@ func NewCrawler(blockchain string, startBlock, finalBlock, confirmations, batchS panic(err) } - client, err := seer_blockchain.NewClient(blockchain, BlockchainURLs[blockchain], timeout) + client, err := seer_blockchain.NewClient(blockchain, seer_blockchain.BlockchainURLs[blockchain], timeout) if err != nil { log.Fatal(err) } diff --git a/indexer/db.go b/indexer/db.go index ca19cc4..f2f7f23 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -77,6 +77,20 @@ func IsBlockchainWithL1Chain(blockchain string) bool { } } +func FilterABIJobs(abiJobs []AbiJob, ids []string) []AbiJob { + var filteredABIJobs []AbiJob + + for _, abiJob := range abiJobs { + for _, id := range ids { + if abiJob.ID == id { + filteredABIJobs = append(filteredABIJobs, abiJob) + } + } + } + + return filteredABIJobs +} + type PostgreSQLpgx struct { pool *pgxpool.Pool } @@ -477,7 +491,7 @@ func (p *PostgreSQLpgx) ReadABIJobs(blockchain string) ([]AbiJob, error) { return nil, nil // or return an appropriate error if this is considered an error state } - log.Println("Parsed abiJobs:", len(abiJobs), "for blockchain:", blockchain) + //log.Println("Parsed abiJobs:", len(abiJobs), "for blockchain:", blockchain) // If you need to process or log the first ABI job separately, do it here return abiJobs, nil @@ -641,7 +655,7 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome } -func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool, outputFilePath string) error { +func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool, outputFilePath string, ids []string) error { pool := p.GetPool() @@ -661,20 +675,30 @@ func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool return err } - log.Println("Found", len(abiJobs), "ABI jobs for blockchain:", blockchain) + if len(ids) > 0 { + abiJobs = FilterABIJobs(abiJobs, ids) + } else { + log.Println("Found", len(abiJobs), "ABI jobs for blockchain:", blockchain) + } + var writer *bufio.Writer + var f *os.File // for each ABI job, check if the selector is correct - f, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if outputFilePath != "" { - if err != nil { - log.Println("Error opening file:", err) - return err - } + f, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + + if err != nil { + log.Println("Error opening file:", err) + return err + } - writer := bufio.NewWriter(f) + writer := bufio.NewWriter(f) - writer.WriteString(fmt.Sprintf("ABI jobs for blockchain: %s runned as WriteToDB: %v recorded at %s\n", blockchain, WriteToDB, time.Now().String())) + writer.WriteString(fmt.Sprintf("ABI jobs for blockchain: %s runned as WriteToDB: %v recorded at %s\n", blockchain, WriteToDB, time.Now().String())) + + } for _, abiJob := range abiJobs { @@ -718,18 +742,25 @@ func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool } - _, err = writer.WriteString(fmt.Sprintf("ABI job ID: %s, Name: %s, Address: %x, Selector: %s, Correct Selector: %s\n", abiJob.ID, abiJob.AbiName, abiJob.Address, abiJob.AbiSelector, selector)) - if err != nil { - log.Println("Error writing to file:", err) - continue + if outputFilePath != "" { + + _, err = writer.WriteString(fmt.Sprintf("ABI job ID: %s, Name: %s, Address: %x, Selector: %s, Correct Selector: %s\n", abiJob.ID, abiJob.AbiName, abiJob.Address, abiJob.AbiSelector, selector)) + if err != nil { + log.Println("Error writing to file:", err) + continue + } + } } } - writer.Flush() - f.Close() + if outputFilePath != "" { + writer.Flush() + + f.Close() + } return nil } @@ -1171,7 +1202,7 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus if !exists { // Initialize the struct if it doesn't exist deployInfo = AbiJobsDeployInfo{ - DeployedBlockNumber: abiJob.DeploymentBlockNumber, + DeployedBlockNumber: *abiJob.DeploymentBlockNumber, IDs: []string{}, } } @@ -1214,100 +1245,6 @@ func (p *PostgreSQLpgx) UpdateAbisAsDone(ids []string) error { return nil } -// func (p *PostgreSQLpgx) SelectAndUpdateUnprocessedAbi(blockchain string, addresses []string, customersIds []string) ([]CustomerUpdates, map[string]uint64, error) { -// pool := p.GetPool() - -// conn, err := pool.Acquire(context.Background()) -// if err != nil { -// return nil, nil, err -// } -// defer conn.Release() - -// var queryBuilder strings.Builder - -// queryArgs := make(pgx.NamedArgs) - -// queryArgs["chain"] = blockchain - -// if len(addresses) > 0 { -// queryBuilder.WriteString(` -// UPDATE abi_jobs -// SET historical_crawl_status = 'pickedup' -// WHERE chain = @chain -// AND address = ANY(@addresses) -// `) -// queryArgs["addresses"] = addresses -// } else { -// queryBuilder.WriteString(` -// UPDATE abi_jobs -// SET historical_crawl_status = 'pickedup' -// WHERE chain = @chain -// AND historical_crawl_status = 'pending' -// AND status = true -// AND deployment_block_number IS NOT NULL -// `) -// } - -// if len(customersIds) > 0 { - -// queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") -// queryArgs["customer_ids"] = customersIds -// } - -// queryBuilder.WriteString(` -// RETURNING id, address, user_id, customer_id, abi_selector, chain, abi_name, status, -// historical_crawl_status, progress, moonworm_task_pickedup, abi, -// (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number -// `) - -// rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) -// if err != nil { -// return nil, nil, err -// } - -// abiJobs, err := pgx.CollectRows(rows, pgx.RowToStructByName[AbiJob]) -// if err != nil { -// log.Println("Error collecting ABI jobs rows", err) -// return nil, nil, err -// } - -// if len(abiJobs) == 0 { -// return []CustomerUpdates{}, map[string]uint64{}, nil -// } - -// customerUpdatesDict := make(map[string]CustomerUpdates) -// addressDeployBlockDict := make(map[string]uint64) - -// for _, abiJob := range abiJobs { -// address := fmt.Sprintf("0x%x", abiJob.Address) - -// if _, exists := customerUpdatesDict[abiJob.CustomerID]; !exists { -// customerUpdatesDict[abiJob.CustomerID] = CustomerUpdates{ -// CustomerID: abiJob.CustomerID, -// Abis: make(map[string]map[string]map[string]string), -// } -// } - -// if _, exists := customerUpdatesDict[abiJob.CustomerID].Abis[address]; !exists { -// customerUpdatesDict[abiJob.CustomerID].Abis[address] = make(map[string]map[string]string) -// } - -// customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = map[string]string{ -// "abi": abiJob.Abi, -// "abi_name": abiJob.AbiName, -// } - -// addressDeployBlockDict[address] = abiJob.DeployedBlockNumber -// } - -// var customerUpdates []CustomerUpdates -// for _, customerUpdate := range customerUpdatesDict { -// customerUpdates = append(customerUpdates, customerUpdate) -// } - -// return customerUpdates, addressDeployBlockDict, nil -// } - func (p *PostgreSQLpgx) FindBatchPath(blockchain string, blockNumber uint64) (string, uint64, uint64, error) { pool := p.GetPool() @@ -1355,28 +1292,32 @@ func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks() (map[string]map[string][ /// get all addresses that not have deploy block number - rows, err := conn.Query(context.Background(), "SELECT id, chain, address FROM abi_jobs WHERE chain=$1 and deploy_block_number is null") + rows, err := conn.Query(context.Background(), "SELECT id, chain, address FROM abi_jobs WHERE deployment_block_number is null") if err != nil { + log.Println("Error querying abi jobs from database", err) return nil, err } // chain, address, ids - var chainsAddresses map[string]map[string][]string + chainsAddresses := make(map[string]map[string][]string) for rows.Next() { var id string var chain string + var raw_address []byte var address string - err = rows.Scan(&id, &chain, &address) + err = rows.Scan(&id, &chain, &raw_address) if err != nil { return nil, err } - if chainsAddresses[chain] == nil { + address = fmt.Sprintf("0x%x", raw_address) + + if _, exists := chainsAddresses[chain]; !exists { chainsAddresses[chain] = make(map[string][]string) } @@ -1384,6 +1325,22 @@ func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks() (map[string]map[string][ } + // Run ensure selector for each chain + + for chain, addressIds := range chainsAddresses { + + for address := range addressIds { + + err := p.EnsureCorrectSelectors(chain, true, "", addressIds[address]) + if err != nil { + + log.Println("Error ensuring correct selectors for chain:", chain, err) + return nil, err + } + } + + } + return chainsAddresses, nil } @@ -1398,9 +1355,16 @@ func (p *PostgreSQLpgx) UpdateAbiJobsDeployBlock(blockNumber uint64, ids []strin defer conn.Release() - // for id, blockNumber := range ids { + // Transform the ids to a slice of UUIDs + idsUUID := make([]uuid.UUID, len(ids)) + for i, id := range ids { + idsUUID[i], err = uuid.Parse(id) + if err != nil { + return err + } + } - _, err = conn.Exec(context.Background(), "UPDATE abi_jobs SET deploy_block_number=$1 WHERE id=$2", blockNumber, ids) + _, err = conn.Exec(context.Background(), "UPDATE abi_jobs SET deployment_block_number=$1 WHERE id=ANY($2)", blockNumber, idsUUID) if err != nil { return err diff --git a/indexer/types.go b/indexer/types.go index 3c21048..7b87491 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -65,7 +65,7 @@ type AbiJob struct { AbiType string CreatedAt time.Time UpdatedAt time.Time - DeploymentBlockNumber uint64 + DeploymentBlockNumber *uint64 } type CustomerUpdates struct { From dd59de68e759f35305deb8263efb44dcf20ec5a9 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 29 Aug 2024 16:12:22 +0300 Subject: [PATCH 03/17] Add fixes. --- blockchain/game7_testnet/game7_testnet.go | 66 +++++++++++------------ indexer/db.go | 26 +++++---- synchronizer/synchronizer.go | 27 +++++----- 3 files changed, 64 insertions(+), 55 deletions(-) diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index ab1cd90..2133551 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -236,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -364,7 +364,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_testnet", @@ -395,14 +394,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7TestnetBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +478,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common. BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +512,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7TestnetBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +633,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7TestnetBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7TestnetBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +650,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -691,9 +690,9 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -743,20 +742,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Get the ABI string contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) if err != nil { - fmt.Println("Error initializing contract ABI: ", err) + fmt.Println("abi", abiMap[e.Address][topicSelector]["abi"]) + fmt.Println("Error initializing contract ABI event: ", err) return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -803,7 +802,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -828,9 +826,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -864,4 +862,4 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil -} \ No newline at end of file +} diff --git a/indexer/db.go b/indexer/db.go index f2f7f23..bd2fe05 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -250,7 +250,7 @@ func (p *PostgreSQLpgx) WriteIndexes(blockchain string, blocksIndexPack []BlockI pool := p.GetPool() conn, err := pool.Acquire(ctx) if err != nil { - fmt.Println("Connection error", err) + log.Println("Connection error", err) return err } defer conn.Release() @@ -304,7 +304,7 @@ func (p *PostgreSQLpgx) executeBatchInsert(tx pgx.Tx, ctx context.Context, table // track execution time if _, err := tx.Exec(ctx, query, valuesSlice...); err != nil { - fmt.Println("Error executing bulk insert", err) + log.Println("Error executing bulk insert", err) return fmt.Errorf("error executing bulk insert for batch: %w", err) } @@ -1144,10 +1144,10 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus queryBuilder.WriteString(` SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, - historical_crawl_status, progress, task_pickedup, abi, + historical_crawl_status, progress, task_pickedup, '[' || abi || ']' as abi, (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number - FROM abi_jobs - WHERE chain = @chain AND historical_crawl_status = 'pickedup' + FROM abi_jobs + WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') `) if len(addresses) > 0 { @@ -1261,14 +1261,22 @@ func (p *PostgreSQLpgx) FindBatchPath(blockchain string, blockNumber uint64) (st var minBlockNumber uint64 var maxBlockNumber uint64 - - query := fmt.Sprintf("SELECT path, min(block_number), max(block_number) FROM %s WHERE block_number = $1", BlocksTableName(blockchain)) - - // Check if we have at least one job before accessing + query := fmt.Sprintf(`WITH path as ( + SELECT + path + from + %s + WHERE + block_number = $1 + ) SELECT path, min(block_number), max(block_number) FROM %s WHERE path = (SELECT path from path) group by path`, BlocksTableName(blockchain), BlocksTableName(blockchain)) err = conn.QueryRow(context.Background(), query, blockNumber).Scan(&path, &minBlockNumber, &maxBlockNumber) if err != nil { + if err == pgx.ErrNoRows { + // Blocks not indexed yet + return "", 0, 0, nil + } return "", 0, 0, diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index dff69c7..60864e0 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -326,6 +326,10 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { // This function will return a list of customer updates 1 update is 1 customer _, lastBlockOfChank, path, updates, err := indexer.DBConnection.ReadUpdates(d.blockchain, d.startBlock, customerIds) + if err != nil { + return isEnd, fmt.Errorf("error reading updates: %w", err) + } + if len(updates) == 0 { log.Printf("No updates found for block %d\n", d.startBlock) return isEnd, nil @@ -420,8 +424,6 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s var useRPC bool var isCycleFinished bool - fmt.Println("HistoricalSyncRef") - // Initialize start block if 0 if d.startBlock == 0 { // Get the latest block from the indexer db @@ -469,11 +471,6 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Main processing loop for { - if d.endBlock != 0 && d.startBlock <= d.endBlock { - isCycleFinished = true - log.Printf("End block %d almost reached", d.endBlock) - } - for address, abisInfo := range addressesAbisInfo { if abisInfo.DeployedBlockNumber > d.startBlock { @@ -504,18 +501,22 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s if path == "" { useRPC = true - d.endBlock = d.startBlock - batchSize } else { d.endBlock = firstBlockOfChunk } - } else { + } + + if useRPC { d.endBlock = d.startBlock - batchSize + if d.endBlock == 0 { + d.endBlock = 1 + } } // Read raw data from storage or via RPC var rawData bytes.Buffer if useRPC { - protoMessage, _, _, err := seer_blockchain.CrawlEntireBlocks(d.Client, big.NewInt(int64(d.startBlock)), big.NewInt(int64(firstBlockOfChunk)), false, 5) + protoMessage, _, _, err := seer_blockchain.CrawlEntireBlocks(d.Client, big.NewInt(int64(d.endBlock)), big.NewInt(int64(d.startBlock)), false, 5) if err != nil { return fmt.Errorf("error reading events via RPC: %w", err) } @@ -538,7 +539,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s } } - log.Printf("Processing %d customer updates for block range %d-%d", len(customerUpdates), d.startBlock, firstBlockOfChunk) + log.Printf("Processing %d customer updates for block range %d-%d", len(customerUpdates), d.startBlock, d.endBlock) // Process customer updates in parallel var wg sync.WaitGroup @@ -585,13 +586,15 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Check for errors from goroutines select { case err := <-errChan: + log.Printf("Error processing customer updates: %v", err) return err default: } d.startBlock = d.endBlock - 1 - if isCycleFinished { + if isCycleFinished || d.startBlock == 0 { + log.Println("Finished processing all customer updates") break } } From 6a15553ea1e8bbfd73efb24e8472676b1b3c60b7 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 31 Aug 2024 23:45:33 +0300 Subject: [PATCH 04/17] Add abi create cli. --- blockchain/arbitrum_one/arbitrum_one.go | 16 +++- .../arbitrum_sepolia/arbitrum_sepolia.go | 16 +++- blockchain/blockchain.go.tmpl | 16 +++- blockchain/ethereum/ethereum.go | 16 +++- .../game7_orbit_arbitrum_sepolia.go | 16 +++- blockchain/game7_testnet/game7_testnet.go | 82 +++++++++++-------- blockchain/imx_zkevm/imx_zkevm.go | 16 +++- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 16 +++- blockchain/mantle/mantle.go | 16 +++- blockchain/mantle_sepolia/mantle_sepolia.go | 16 +++- blockchain/polygon/polygon.go | 16 +++- blockchain/sepolia/sepolia.go | 16 +++- blockchain/xai/xai.go | 16 +++- blockchain/xai_sepolia/xai_sepolia.go | 16 +++- cmd.go | 71 +++++++++++++++- indexer/db.go | 73 +++++++++++++++++ synchronizer/synchronizer.go | 6 +- 17 files changed, 389 insertions(+), 51 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 95cccef..77bd9ee 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index a92107f..1fe0bef 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 94cd1c0..4309e58 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 6d50b50..7da3630 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 37c2993..2315f07 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 2133551..d71b0f7 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -236,7 +236,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -364,6 +364,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_testnet", @@ -394,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7TestnetBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -478,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common. BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -512,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7TestnetBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -633,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7TestnetBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7TestnetBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -650,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -684,19 +685,33 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -742,20 +757,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma // Get the ABI string contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) if err != nil { - fmt.Println("abi", abiMap[e.Address][topicSelector]["abi"]) - fmt.Println("Error initializing contract ABI event: ", err) + fmt.Println("Error initializing contract ABI: ", err) return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -802,6 +817,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -826,9 +842,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -862,4 +878,4 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil -} +} \ No newline at end of file diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 06464b8..a2bb5b8 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 408619b..e7c942d 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index cc4cb71..855714e 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index 53c291b..26f1502 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 0bb98ad..2db5432 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index bb12b29..142b291 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index ff88b5e..cbab6cc 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 7387848..d5c2765 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -685,7 +685,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -698,6 +698,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) diff --git a/cmd.go b/cmd.go index c3ce596..602f85c 100644 --- a/cmd.go +++ b/cmd.go @@ -244,6 +244,11 @@ func CreateCrawlerCommand() *cobra.Command { return crawlerErr } + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -315,6 +320,11 @@ func CreateSynchronizerCommand() *cobra.Command { return syncErr } + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + if chain == "" { return fmt.Errorf("blockchain is required via --chain") } @@ -788,7 +798,63 @@ func CreateDatabaseOperationCommand() *cobra.Command { }, } + var jobChain, address, abiFile, customerId, userId string + + createJobsCommand := &cobra.Command{ + Use: "create-jobs", + Short: "Create jobs for ABI", + PreRunE: func(cmd *cobra.Command, args []string) error { + indexerErr := indexer.CheckVariablesForIndexer() + if indexerErr != nil { + return indexerErr + } + + indexer.InitDBConnection() + + blockchainErr := seer_blockchain.CheckVariablesForBlockchains() + if blockchainErr != nil { + return blockchainErr + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + + // check if the chain is supported + if _, ok := seer_blockchain.BlockchainURLs[jobChain]; !ok { + return fmt.Errorf("chain %s is not supported", jobChain) + } + + client, clientErr := seer_blockchain.NewClient(jobChain, seer_blockchain.BlockchainURLs[jobChain], 30) + if clientErr != nil { + return clientErr + } + + // detect deploy block + + deploy_block, deployErr := seer_blockchain.FindDeployedBlock(client, address) + + if deployErr != nil { + return deployErr + } + + createJobsErr := indexer.DBConnection.CreateJobsFromAbi(jobChain, address, abiFile, customerId, userId, deploy_block) + if createJobsErr != nil { + return createJobsErr + } + + return nil + }, + } + + createJobsCommand.Flags().StringVar(&jobChain, "jobChain", "ethereum", "The blockchain to crawl (default: ethereum)") + createJobsCommand.Flags().StringVar(&address, "address", "", "The address to create jobs for") + createJobsCommand.Flags().StringVar(&abiFile, "abi-file", "", "The path to the ABI file") + createJobsCommand.Flags().StringVar(&customerId, "customer-id", "", "The customer ID to create jobs for (default: '')") + createJobsCommand.Flags().StringVar(&userId, "user-id", "00000000-0000-0000-0000-000000000000", "The user ID to create jobs for (default: '00000000-0000-0000-0000-000000000000')") + indexCommand.AddCommand(deploymentBlocksCommand) + indexCommand.AddCommand(createJobsCommand) databaseCmd.AddCommand(indexCommand) return databaseCmd @@ -799,7 +865,7 @@ func CreateHistoricalSyncCommand() *cobra.Command { var chain, baseDir, customerDbUriFlag string var addresses, customerIds []string var startBlock, endBlock, batchSize uint64 - var timeout int + var timeout, threads int var auto bool historicalSyncCmd := &cobra.Command{ @@ -840,7 +906,7 @@ func CreateHistoricalSyncCommand() *cobra.Command { return synchonizerErr } - err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, customerIds, addresses, batchSize, auto) + err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, customerIds, addresses, batchSize, auto, threads) if err != nil { return err @@ -860,6 +926,7 @@ func CreateHistoricalSyncCommand() *cobra.Command { historicalSyncCmd.Flags().StringSliceVar(&customerIds, "customer-ids", []string{}, "The list of customer IDs to sync") historicalSyncCmd.Flags().StringSliceVar(&addresses, "addresses", []string{}, "The list of addresses to sync") historicalSyncCmd.Flags().BoolVar(&auto, "auto", false, "Set this flag to sync all unfinished historical crawl from the database (default: false)") + historicalSyncCmd.Flags().IntVar(&threads, "threads", 5, "Number of go-routines for concurrent crawling (default: 5)") return historicalSyncCmd } diff --git a/indexer/db.go b/indexer/db.go index bd2fe05..f980f75 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -4,7 +4,9 @@ import ( "bufio" "context" "encoding/hex" + "encoding/json" "fmt" + "io/ioutil" "log" "os" "strconv" @@ -1381,3 +1383,74 @@ func (p *PostgreSQLpgx) UpdateAbiJobsDeployBlock(blockNumber uint64, ids []strin return nil } + +func (p *PostgreSQLpgx) CreateJobsFromAbi(chain string, address string, abiFile string, customerID string, userID string, deployBlock uint64) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + if err != nil { + return err + } + defer conn.Release() + + abiData, err := ioutil.ReadFile(abiFile) + if err != nil { + return err + } + + var abiJson []map[string]interface{} + err = json.Unmarshal(abiData, &abiJson) + if err != nil { + return err + } + + for _, abiJob := range abiJson { + + // Generate a new UUID for the id column + jobID := uuid.New() + + abiJobJson, err := json.Marshal(abiJob) + if err != nil { + log.Println("Error marshalling ABI job to JSON:", abiJob, err) + return err + } + + // Wrap the JSON string in an array + abiJsonArray := "[" + string(abiJobJson) + "]" + + // Get the correct selector for the ABI + abiObj, err := abi.JSON(strings.NewReader(abiJsonArray)) + if err != nil { + log.Println("Error parsing ABI for ABI job:", abiJsonArray, err) + return err + } + var selector string + + if abiJob["type"] == "event" { + selector = abiObj.Events[abiJob["name"].(string)].ID.String() + } else if abiJob["type"] == "function" { + selectorRaw := abiObj.Methods[abiJob["name"].(string)].ID + selector = fmt.Sprintf("0x%x", selectorRaw) + } else { + log.Println("ABI type not supported:", abiJob["type"]) + continue + } + + addressBytes, err := decodeAddress(address) + + if err != nil { + log.Println("Error decoding address:", err, address) + continue + } + + _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) + + if err != nil { + return err + } + + } + + return nil + +} diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 60864e0..9f63011 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -420,7 +420,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, nil } -func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool) error { +func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool, threads int) error { var useRPC bool var isCycleFinished bool @@ -543,8 +543,8 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Process customer updates in parallel var wg sync.WaitGroup - sem := make(chan struct{}, 5) // Semaphore to control concurrency - errChan := make(chan error, 1) // Buffered channel for error handling + sem := make(chan struct{}, threads) // Semaphore to control concurrency + errChan := make(chan error, 1) // Buffered channel for error handling for _, update := range customerUpdates { wg.Add(1) From ffdf690938d5ba6acba157392136edac45e9f432 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 11 Sep 2024 16:03:00 +0300 Subject: [PATCH 05/17] Add get RPC labels methods. Partial refactor of synchronizer. --- blockchain/arbitrum_one/arbitrum_one.go | 386 +++++++++++++++++- .../arbitrum_sepolia/arbitrum_sepolia.go | 386 +++++++++++++++++- blockchain/blockchain.go.tmpl | 386 +++++++++++++++++- blockchain/ethereum/ethereum.go | 386 +++++++++++++++++- .../game7_orbit_arbitrum_sepolia.go | 386 +++++++++++++++++- blockchain/game7_testnet/game7_testnet.go | 386 +++++++++++++++++- blockchain/handlers.go | 2 + blockchain/imx_zkevm/imx_zkevm.go | 386 +++++++++++++++++- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 386 +++++++++++++++++- blockchain/mantle/mantle.go | 386 +++++++++++++++++- blockchain/mantle_sepolia/mantle_sepolia.go | 386 +++++++++++++++++- blockchain/polygon/polygon.go | 298 ++++++++++++-- blockchain/sepolia/sepolia.go | 386 +++++++++++++++++- blockchain/xai/xai.go | 386 +++++++++++++++++- blockchain/xai_sepolia/xai_sepolia.go | 386 +++++++++++++++++- indexer/db.go | 1 + indexer/types.go | 11 - synchronizer/synchronizer.go | 341 ++++++++++------ 18 files changed, 5434 insertions(+), 237 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 77bd9ee..cb47c36 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index 1fe0bef..f636ef5 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 4309e58..4103158 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 7da3630..617e60d 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 2315f07..85c69b1 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index d71b0f7..0157f5e 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/handlers.go b/blockchain/handlers.go index 13e6eab..030fdc9 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -92,6 +92,8 @@ type BlockchainClient interface { DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) ChainType() string GetCode(context.Context, common.Address, uint64) ([]byte, error) + GetTransactionsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) + GetEventsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.EventLabel, error) } func GetLatestBlockNumberWithRetry(client BlockchainClient, retryAttempts int, retryWaitTime time.Duration) (*big.Int, error) { diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index a2bb5b8..5439df2 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index e7c942d..9718603 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 855714e..ff5a53c 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index 26f1502..c6508c8 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 2db5432..dfc4a59 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -236,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -364,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("polygon", @@ -395,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &PolygonBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -512,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *PolygonBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -634,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*PolygonBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch PolygonBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -817,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -842,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -878,4 +866,234 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil -} \ No newline at end of file +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + +} diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index 142b291..e0bbf74 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index cbab6cc..40a5be0 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index d5c2765..5e4b085 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -72,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -207,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -878,4 +879,381 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil +} + + +// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +// var labels []indexer.TransactionLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, transaction := range transactions { +// var transactionJson seer_common.TransactionJson +// err := json.Unmarshal([]byte(transaction), &transactionJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// selector := transactionJson.Input[:10] + +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) + +// if err != nil { +// return nil, err +// } + +// inputData, err := hex.DecodeString(transactionJson.Input[2:]) +// if err != nil { +// fmt.Println("Error decoding input data: ", err) +// return nil, err +// } + +// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + +// if decodeErr != nil { +// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": transactionJson, +// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], +// "selector": selector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert JSON byte slice to string +// labelDataString := string(labelDataBytes) + +// // Convert transaction to label +// transactionLabel := indexer.TransactionLabel{ +// Address: transactionJson.ToAddress, +// BlockNumber: transactionJson.BlockNumber, +// BlockHash: transactionJson.BlockHash, +// CallerAddress: transactionJson.FromAddress, +// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], +// LabelType: "tx_call", +// OriginAddress: transactionJson.FromAddress, +// Label: label, +// TransactionHash: transactionJson.Hash, +// LabelData: labelDataString, +// BlockTimestamp: transactionJson.BlockTimestamp, +// } + +// labels = append(labels, transactionLabel) + +// } + +// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +// var labels []indexer.EventLabel +// var decodedArgs map[string]interface{} +// var decodeErr error + +// for _, event := range events { +// var eventJson seer_common.EventJson +// err := json.Unmarshal([]byte(event), &eventJson) +// if err != nil { +// return nil, err +// } + +// label := indexer.SeerCrawlerLabel + +// var topicSelector string + +// if len(eventJson.Topics) > 0 { +// topicSelector = eventJson.Topics[0] +// } else { +// // 0x0 is the default topic selector +// topicSelector = "0x0" +// } + +// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { +// continue +// } + +// // Get the ABI string +// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) +// if err != nil { +// fmt.Println("Error initializing contract ABI: ", err) +// return nil, err +// } + +// // Decode the event data +// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) +// if decodeErr != nil { +// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) +// decodedArgs = map[string]interface{}{ +// "input_raw": eventJson, +// "abi": abiMap[eventJson.Address][topicSelector]["abi"], +// "selector": topicSelector, +// "error": decodeErr, +// } +// label = indexer.SeerCrawlerRawLabel +// } + +// // Convert decodedArgs map to JSON +// labelDataBytes, err := json.Marshal(decodedArgs) +// if err != nil { +// fmt.Println("Error converting decodedArgs to JSON: ", err) +// return nil, err +// } + +// // Convert event to label +// eventLabel := indexer.EventLabel{ +// Label: label, +// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], +// LabelType: "event", +// BlockNumber: eventJson.BlockNumber, +// BlockHash: eventJson.BlockHash, +// Address: eventJson.Address, +// OriginAddress: eventJson.OriginAddress, +// TransactionHash: eventJson.TransactionHash, +// LabelData: string(labelDataBytes), // Convert JSON byte slice to string +// BlockTimestamp: eventJson.BlockTimestamp, +// LogIndex: eventJson.LogIndex, +// } + +// labels = append(labels, eventLabel) + +// } + +// return labels, nil +// } + + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { + var transactionsLabels []indexer.TransactionLabel + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, err + } + + // Get transactions in range + + for _, block := range blocks { + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: uintBlockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + // Get events in range + + var addresses []common.Address + var topics [][]common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + addresses = append(addresses, common.HexToAddress(address)) + topics = append(topics, []common.Hash{common.HexToHash(selector)}) + } + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: topics, + } + + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + // get transaction information + transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) + + if err != nil { + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + if err != nil { + return nil, err + } + + logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + } \ No newline at end of file diff --git a/indexer/db.go b/indexer/db.go index f980f75..6d325a6 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -1197,6 +1197,7 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = map[string]string{ "abi": abiJob.Abi, "abi_name": abiJob.AbiName, + "abi_type": abiJob.AbiType, } // Retrieve the struct from the map diff --git a/indexer/types.go b/indexer/types.go index 7b87491..305edaa 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -76,26 +76,15 @@ type CustomerUpdates struct { } type TaskForTransaction struct { - Hash string `json:"hash"` Address string `json:"address"` Selector string `json:"selector"` ABI string `json:"abi"` - RowID uint64 `json:"row_id"` - Path string `json:"path"` } type TaskForLog struct { // Assuming structure similar to TransactionIndex - Hash string `json:"hash"` Address string `json:"address"` Selector string `json:"selector"` ABI string `json:"abi"` - RowID uint64 `json:"row_id"` - Path string `json:"path"` -} - -type RawChainData struct { - Transactions []TaskForTransaction `json:"transactions"` - Events []TaskForLog `json:"events"` } type EventLabel struct { diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 9f63011..20976fa 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "log" - "math/big" "net/http" "net/url" "path/filepath" @@ -18,8 +17,6 @@ import ( "github.com/moonstream-to/seer/crawler" "github.com/moonstream-to/seer/indexer" "github.com/moonstream-to/seer/storage" - "golang.org/x/exp/slices" - "google.golang.org/protobuf/proto" ) type Synchronizer struct { @@ -157,6 +154,60 @@ type CustomerDBConnection struct { Pgx *indexer.PostgreSQLpgx } +func CustomersLatestBlocks(customerDBConnections map[string]CustomerDBConnection, blockchain string) (map[string]uint64, error) { + latestBlocks := make(map[string]uint64) + for id, customer := range customerDBConnections { + pool := customer.Pgx.GetPool() + conn, err := pool.Acquire(context.Background()) + if err != nil { + log.Println("Error acquiring pool connection: ", err) + return nil, err + } + defer conn.Release() + + latestLabelBlock, err := customer.Pgx.ReadLastLabel(blockchain) + if err != nil { + log.Println("Error reading latest block: ", err) + return nil, err + } + latestBlocks[id] = latestLabelBlock + log.Printf("Latest block for customer %s is: %d\n", id, latestLabelBlock) + } + return latestBlocks, nil +} + +func (d *Synchronizer) StartBlockLookUp(customerDBConnections map[string]CustomerDBConnection, blockchain string, blockShift int64) (uint64, error) { + var startBlock uint64 + latestCustomerBlocks, err := CustomersLatestBlocks(customerDBConnections, d.blockchain) + if err != nil { + return startBlock, fmt.Errorf("error getting latest blocks for customers: %w", err) + } + + // Determine the start block as the maximum of the latest blocks of all customers + var maxCustomerLatestBlock uint64 + if len(latestCustomerBlocks) > 0 { + for _, block := range latestCustomerBlocks { + if block > maxCustomerLatestBlock { + maxCustomerLatestBlock = block + } + } + } + + if maxCustomerLatestBlock != 0 { + startBlock = maxCustomerLatestBlock + } else { + // In case start block is still 0, get the latest block from the blockchain minus shift + latestBlockNumber, latestErr := d.Client.GetLatestBlockNumber() + if latestErr != nil { + return startBlock, fmt.Errorf("error getting latest block number: %w", latestErr) + } + startBlock = uint64(latestBlockNumber.Int64() - blockShift) + } + + return startBlock, nil + +} + // getCustomers fetches ABI jobs, extracts customer IDs, and establishes database connections. func (d *Synchronizer) getCustomers(customerDbUriFlag string, customerIds []string) (map[string]CustomerDBConnection, []string, error) { customerDBConnections := make(map[string]CustomerDBConnection) @@ -249,43 +300,13 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, customersErr } - // Set startBlocks as latest labeled block from customers or from the blockchain if customers are not indexed yet + // Set start block if 0 if d.startBlock == 0 { - var latestCustomerBlocks []uint64 - for id, customer := range customerDBConnections { - - pool := customer.Pgx.GetPool() - conn, err := pool.Acquire(context.Background()) - if err != nil { - log.Println("Error acquiring pool connection: ", err) - return isEnd, err - } - defer conn.Release() - - latestLabelBlock, err := customer.Pgx.ReadLastLabel(d.blockchain) - if err != nil { - log.Println("Error reading latest block: ", err) - return isEnd, err - } - latestCustomerBlocks = append(latestCustomerBlocks, latestLabelBlock) - log.Printf("Latest block for customer %s is: %d\n", id, latestLabelBlock) - } - - // Determine the start block as the maximum of the latest blocks of all customers - var maxCustomerLatestBlock uint64 - if len(latestCustomerBlocks) != 0 { - maxCustomerLatestBlock = slices.Max(latestCustomerBlocks) - } - if maxCustomerLatestBlock != 0 { - d.startBlock = maxCustomerLatestBlock - 100 - } else { - // In case start block is still 0, get the latest block from the blockchain minus shift - latestBlockNumber, latestErr := d.Client.GetLatestBlockNumber() - if latestErr != nil { - return isEnd, fmt.Errorf("failed to get latest block number: %v", latestErr) - } - d.startBlock = uint64(latestBlockNumber.Int64() - crawler.SeerDefaultBlockShift) + startBlock, startErr := d.StartBlockLookUp(customerDBConnections, d.blockchain, crawler.SeerDefaultBlockShift) + if startErr != nil { + return isEnd, fmt.Errorf("error determining start block: %w", startErr) } + d.startBlock = startBlock } // Get the latest block from the indexer db @@ -294,7 +315,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, idxLatestErr } - if d.endBlock != 0 && indexedLatestBlock > d.endBlock { + if d.endBlock > 0 && indexedLatestBlock > d.endBlock { indexedLatestBlock = d.endBlock } @@ -311,13 +332,14 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { //var noUpdatesFoundErr *indexer.NoUpdatesFoundError var isCycleFinished bool for { - if d.endBlock != 0 { - if d.startBlock >= d.endBlock { - isEnd = true - isCycleFinished = true - log.Printf("End block %d almost reached", d.endBlock) - } + // Check if end block is reached or start block exceeds end block + if d.endBlock > 0 && d.startBlock >= d.endBlock { + isEnd = true + isCycleFinished = true + log.Printf("End block %d almost reached", d.endBlock) + break } + if d.endBlock >= indexedLatestBlock { isCycleFinished = true } @@ -325,7 +347,6 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { // Read updates from the indexer db // This function will return a list of customer updates 1 update is 1 customer _, lastBlockOfChank, path, updates, err := indexer.DBConnection.ReadUpdates(d.blockchain, d.startBlock, customerIds) - if err != nil { return isEnd, fmt.Errorf("error reading updates: %w", err) } @@ -338,7 +359,6 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { if crawler.SEER_CRAWLER_DEBUG { log.Printf("Read batch key: %s", path) } - log.Println("Last block of current chank: ", lastBlockOfChank) // Read the raw data from the storage for current path @@ -347,10 +367,6 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, fmt.Errorf("error reading events for customers %s: %w", readErr) } - if err != nil { - return isEnd, fmt.Errorf("error reading updates: %w", err) - } - log.Printf("Read %d users updates from the indexer db in range of blocks %d-%d\n", len(updates), d.startBlock, lastBlockOfChank) var wg sync.WaitGroup @@ -359,45 +375,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { for _, update := range updates { wg.Add(1) - go func(update indexer.CustomerUpdates, rawData bytes.Buffer) { - defer wg.Done() - - sem <- struct{}{} // Acquire semaphore - - // Get the RDS connection for the customer - customer, customerExists := customerDBConnections[update.CustomerID] - if !customerExists { - errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) - return - } - - // Create a connection to the user RDS - pool := customer.Pgx.GetPool() - conn, err := pool.Acquire(context.Background()) - if err != nil { - errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) - return - } - defer conn.Release() - - var decodedEventsPack []indexer.EventLabel - var decodedTransactionsPack []indexer.TransactionLabel - - // decodedEvents, decodedTransactions, decErr - decodedEvents, decodedTransactions, decErr := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis) - if decErr != nil { - log.Println("Error decoding events: ", decErr) - errChan <- fmt.Errorf("error decoding events for customer %s: %w", update.CustomerID, decErr) - return - } - - decodedEventsPack = append(decodedEventsPack, decodedEvents...) - decodedTransactionsPack = append(decodedTransactionsPack, decodedTransactions...) - - customer.Pgx.WriteLabes(d.blockchain, decodedTransactionsPack, decodedEventsPack) - - <-sem - }(update, rawData) + go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) } wg.Wait() @@ -434,7 +412,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s d.startBlock = indexedLatestBlock } - // Automatically update ABI jobs if auto mode is enabled + // Automatically update ABI jobs as active if auto mode is enabled if auto { if err := indexer.DBConnection.UpdateAbiJobsStatus(d.blockchain); err != nil { return fmt.Errorf("error updating ABI: %w", err) @@ -516,22 +494,25 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Read raw data from storage or via RPC var rawData bytes.Buffer if useRPC { - protoMessage, _, _, err := seer_blockchain.CrawlEntireBlocks(d.Client, big.NewInt(int64(d.endBlock)), big.NewInt(int64(d.startBlock)), false, 5) - if err != nil { - return fmt.Errorf("error reading events via RPC: %w", err) - } + // protoMessage, _, _, err := seer_blockchain.CrawlEntireBlocks(d.Client, big.NewInt(int64(d.endBlock)), big.NewInt(int64(d.startBlock)), false, 5) + // if err != nil { + // return fmt.Errorf("error reading events via RPC: %w", err) + // } - blocksBatch, err := d.Client.ProcessBlocksToBatch(protoMessage) - if err != nil { - return fmt.Errorf("error processing blocks to batch: %w", err) - } + // blocksBatch, err := d.Client.ProcessBlocksToBatch(protoMessage) + // if err != nil { + // return fmt.Errorf("error processing blocks to batch: %w", err) + // } - dataBytes, err := proto.Marshal(blocksBatch) - if err != nil { - return fmt.Errorf("error marshaling protoMessage: %w", err) - } + // dataBytes, err := proto.Marshal(blocksBatch) + // if err != nil { + // return fmt.Errorf("error marshaling protoMessage: %w", err) + // } + + // rawData = *bytes.NewBuffer(dataBytes) + + // Read transactions and events from the blockchain - rawData = *bytes.NewBuffer(dataBytes) } else { rawData, err = d.StorageInstance.Read(path) if err != nil { @@ -548,36 +529,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s for _, update := range customerUpdates { wg.Add(1) - go func(update indexer.CustomerUpdates) { - defer wg.Done() - sem <- struct{}{} // Acquire semaphore - - customer, exists := customerDBConnections[update.CustomerID] - if !exists { - errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) - <-sem // Release semaphore - return - } - - conn, err := customer.Pgx.GetPool().Acquire(context.Background()) - if err != nil { - errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) - <-sem // Release semaphore - return - } - defer conn.Release() - - decodedEvents, decodedTransactions, err := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis) - if err != nil { - errChan <- fmt.Errorf("error decoding events for customer %s: %w", update.CustomerID, err) - <-sem // Release semaphore - return - } - - customer.Pgx.WriteLabes(d.blockchain, decodedTransactions, decodedEvents) - - <-sem // Release semaphore - }(update) + go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) } wg.Wait() @@ -601,3 +553,124 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s return nil } + +func (d *Synchronizer) processProtoCustomerUpdate( + update indexer.CustomerUpdates, + rawData bytes.Buffer, + customerDBConnections map[string]CustomerDBConnection, + sem chan struct{}, + errChan chan error, + wg *sync.WaitGroup, +) { + // Decode input raw proto data using ABIs + // Write decoded data to the user Database + + defer wg.Done() + sem <- struct{}{} // Acquire semaphore + + customer, exists := customerDBConnections[update.CustomerID] + if !exists { + errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) + <-sem // Release semaphore + return + } + + conn, err := customer.Pgx.GetPool().Acquire(context.Background()) + if err != nil { + errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + defer conn.Release() + + decodedEvents, decodedTransactions, err := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis) + if err != nil { + errChan <- fmt.Errorf("error decoding events for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + + customer.Pgx.WriteLabes(d.blockchain, decodedTransactions, decodedEvents) + + <-sem // Release semaphore +} + +func (d *Synchronizer) processRPCCustomerUpdate( + update indexer.CustomerUpdates, + customerDBConnections map[string]CustomerDBConnection, + sem chan struct{}, + errChan chan error, + wg *sync.WaitGroup, +) { + // Decode input raw proto data using ABIs + // Write decoded data to the user Database + + defer wg.Done() + sem <- struct{}{} // Acquire semaphore + + customer, exists := customerDBConnections[update.CustomerID] + if !exists { + errChan <- fmt.Errorf("no DB connection for customer %s", update.CustomerID) + <-sem // Release semaphore + return + } + + conn, err := customer.Pgx.GetPool().Acquire(context.Background()) + if err != nil { + errChan <- fmt.Errorf("error acquiring connection for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + defer conn.Release() + + // split abis by the type of the task + + eventAbis := make(map[string]map[string]map[string]string) + transactionAbis := make(map[string]map[string]map[string]string) + + for address, selectorMap := range update.Abis { + + for selector, abiInfo := range selectorMap { + if abiInfo["type"] == "event" { + if _, ok := eventAbis[address]; !ok { + eventAbis[address] = make(map[string]map[string]string) + } + eventAbis[address][selector] = abiInfo + } else { + if _, ok := transactionAbis[address]; !ok { + transactionAbis[address] = make(map[string]map[string]string) + } + transactionAbis[address][selector] = abiInfo + } + } + + } + + // Read transactions and events from the blockchain + + // Transactions + + transactions, err := d.Client.GetTransactionsLabels(d.startBlock, d.endBlock, transactionAbis) + + if err != nil { + errChan <- fmt.Errorf("error getting transactions for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + + // Events + + events, err := d.Client.GetEventsLabels(d.startBlock, d.endBlock, eventAbis) + + if err != nil { + + errChan <- fmt.Errorf("error getting events for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + + } + + customer.Pgx.WriteLabes(d.blockchain, transactions, events) + + <-sem // Release semaphore +} From a25e1f4d97b43fa6de2e75830783c6ad47c8be2c Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 11 Sep 2024 16:05:47 +0300 Subject: [PATCH 06/17] Go formating --- blockchain/arbitrum_one/arbitrum_one.go | 69 +++++++++---------- .../arbitrum_sepolia/arbitrum_sepolia.go | 69 +++++++++---------- blockchain/ethereum/ethereum.go | 63 +++++++---------- .../game7_orbit_arbitrum_sepolia.go | 69 +++++++++---------- blockchain/game7_testnet/game7_testnet.go | 69 +++++++++---------- blockchain/imx_zkevm/imx_zkevm.go | 63 +++++++---------- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 63 +++++++---------- blockchain/mantle/mantle.go | 63 +++++++---------- blockchain/mantle_sepolia/mantle_sepolia.go | 63 +++++++---------- blockchain/sepolia/sepolia.go | 63 +++++++---------- blockchain/xai/xai.go | 69 +++++++++---------- blockchain/xai_sepolia/xai_sepolia.go | 69 +++++++++---------- go.mod | 2 +- 13 files changed, 337 insertions(+), 457 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index cb47c36..40039a7 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_one", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumOneBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.B BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumOneBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumOneBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumOneBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +878,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1023,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1220,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1251,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index f636ef5..641bcc2 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +878,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1023,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1220,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1251,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 617e60d..bfbc7d0 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("ethereum", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &EthereumBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *EthereumBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*EthereumBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch EthereumBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +868,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1013,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1210,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1241,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 85c69b1..a434a3a 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_orbit_arbitrum_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7OrbitArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7OrbitArbitrumSepoliaBl TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7OrbitArbitrumSepoliaB func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7OrbitArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +878,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1023,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1220,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1251,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 0157f5e..7911b46 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_testnet", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7TestnetBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common. BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7TestnetBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7TestnetBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7TestnetBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +878,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1023,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1220,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1251,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 5439df2..79cfa2b 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +868,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1013,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1210,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1241,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 9718603..30cae97 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +868,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1013,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1210,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1241,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index ff5a53c..fd37ca0 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.Blocks BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +868,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1013,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1210,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1241,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index c6508c8..537ab1e 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleSepoliaBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +868,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1013,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1210,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1241,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index e0bbf74..6618085 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &SepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *SepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*SepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch SepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +868,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1013,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1210,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1241,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 40a5be0..248b668 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBat BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +878,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1023,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1220,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1251,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 5e4b085..68290d3 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.Bl BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiSepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -881,7 +878,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - // func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { // var labels []indexer.TransactionLabel // var decodedArgs map[string]interface{} @@ -1027,7 +1023,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa // return labels, nil // } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) @@ -1225,7 +1220,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) + blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) if err != nil { return nil, err } @@ -1256,4 +1251,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 8c72727..0c7c138 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/jackc/pgx/v5 v5.5.3 github.com/spf13/cobra v1.8.0 golang.org/x/crypto v0.20.0 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/term v0.17.0 golang.org/x/tools v0.15.0 google.golang.org/api v0.167.0 @@ -62,6 +61,7 @@ require ( go.opentelemetry.io/otel v1.23.0 // indirect go.opentelemetry.io/otel/metric v1.23.0 // indirect go.opentelemetry.io/otel/trace v1.23.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect From 2ead657a2051f9895a5455fae4a88b7444ed7a77 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 13 Sep 2024 19:38:34 +0300 Subject: [PATCH 07/17] Add refactoring on caching. --- blockchain/arbitrum_one/arbitrum_one.go | 250 +++++------------ .../arbitrum_sepolia/arbitrum_sepolia.go | 250 +++++------------ blockchain/blockchain.go.tmpl | 254 +++++------------- blockchain/common/decoding.go | 7 + blockchain/ethereum/ethereum.go | 250 +++++------------ .../game7_orbit_arbitrum_sepolia.go | 250 +++++------------ blockchain/game7_testnet/game7_testnet.go | 250 +++++------------ blockchain/handlers.go | 4 +- blockchain/imx_zkevm/imx_zkevm.go | 250 +++++------------ .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 250 +++++------------ blockchain/mantle/mantle.go | 250 +++++------------ blockchain/mantle_sepolia/mantle_sepolia.go | 250 +++++------------ blockchain/polygon/polygon.go | 105 +++++--- blockchain/sepolia/sepolia.go | 250 +++++------------ blockchain/xai/xai.go | 250 +++++------------ blockchain/xai_sepolia/xai_sepolia.go | 250 +++++------------ cmd.go | 2 +- indexer/db.go | 27 +- synchronizer/synchronizer.go | 69 +++-- 19 files changed, 1055 insertions(+), 2413 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 40039a7..dbfc662 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index 641bcc2..bf8bac9 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 4103158..80cfc05 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -881,172 +881,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } - -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1054,6 +928,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1063,13 +939,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1088,7 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1101,17 +977,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1126,7 +992,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1136,23 +1002,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1160,9 +1031,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1213,24 +1083,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1246,7 +1132,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } @@ -1256,4 +1142,4 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return eventsLabels, nil -} \ No newline at end of file +} diff --git a/blockchain/common/decoding.go b/blockchain/common/decoding.go index ff41dca..a4008f0 100644 --- a/blockchain/common/decoding.go +++ b/blockchain/common/decoding.go @@ -101,6 +101,13 @@ type QueryFilter struct { Topics [][]string `json:"topics"` } +type BlockWithTransactions struct { + BlockNumber uint64 + BlockHash string + BlockTimestamp uint64 + Transactions map[string]TransactionJson +} + // ReadJsonBlocks reads blocks from a JSON file func ReadJsonBlocks() []*BlockJson { file, err := os.Open("data/blocks_go.json") diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index bfbc7d0..0d7190c 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index a434a3a..f25885a 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 7911b46..8f98bda 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/handlers.go b/blockchain/handlers.go index 030fdc9..94f6ca4 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -92,8 +92,8 @@ type BlockchainClient interface { DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) ChainType() string GetCode(context.Context, common.Address, uint64) ([]byte, error) - GetTransactionsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) - GetEventsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.EventLabel, error) + GetTransactionsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) + GetEventsLabels(uint64, uint64, map[string]map[string]map[string]string, map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) } func GetLatestBlockNumberWithRetry(client BlockchainClient, retryAttempts int, retryWaitTime time.Duration) (*big.Int, error) { diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 79cfa2b..feae909 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 30cae97..9fb847b 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index fd37ca0..78cded5 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index 537ab1e..c7c87e8 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index dfc4a59..335c83a 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -874,19 +874,40 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -894,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -903,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -928,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -941,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -966,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -976,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1000,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1053,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1086,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index 6618085..d4c116d 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -868,170 +868,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1039,6 +915,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1048,13 +926,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1073,7 +951,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1086,17 +964,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1111,7 +979,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1121,23 +989,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1145,9 +1018,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1198,24 +1070,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1231,7 +1119,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 248b668..903e27e 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 68290d3..7403f4d 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -878,170 +878,46 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return labels, nil } -// func DecodedJsonTransactionsToLabels(transactions []string, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { -// var labels []indexer.TransactionLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, transaction := range transactions { -// var transactionJson seer_common.TransactionJson -// err := json.Unmarshal([]byte(transaction), &transactionJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// selector := transactionJson.Input[:10] - -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[transactionJson.ToAddress][selector]["abi"])) - -// if err != nil { -// return nil, err -// } - -// inputData, err := hex.DecodeString(transactionJson.Input[2:]) -// if err != nil { -// fmt.Println("Error decoding input data: ", err) -// return nil, err -// } - -// decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) - -// if decodeErr != nil { -// fmt.Println("Error decoding transaction not decoded data: ", transactionJson.Hash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": transactionJson, -// "abi": abiMap[transactionJson.ToAddress][selector]["abi"], -// "selector": selector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert JSON byte slice to string -// labelDataString := string(labelDataBytes) - -// // Convert transaction to label -// transactionLabel := indexer.TransactionLabel{ -// Address: transactionJson.ToAddress, -// BlockNumber: transactionJson.BlockNumber, -// BlockHash: transactionJson.BlockHash, -// CallerAddress: transactionJson.FromAddress, -// LabelName: abiMap[transactionJson.ToAddress][selector]["abi_name"], -// LabelType: "tx_call", -// OriginAddress: transactionJson.FromAddress, -// Label: label, -// TransactionHash: transactionJson.Hash, -// LabelData: labelDataString, -// BlockTimestamp: transactionJson.BlockTimestamp, -// } - -// labels = append(labels, transactionLabel) - -// } - -// func DecodeJsonEventsToLabels( events []string, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { -// var labels []indexer.EventLabel -// var decodedArgs map[string]interface{} -// var decodeErr error - -// for _, event := range events { -// var eventJson seer_common.EventJson -// err := json.Unmarshal([]byte(event), &eventJson) -// if err != nil { -// return nil, err -// } - -// label := indexer.SeerCrawlerLabel - -// var topicSelector string - -// if len(eventJson.Topics) > 0 { -// topicSelector = eventJson.Topics[0] -// } else { -// // 0x0 is the default topic selector -// topicSelector = "0x0" -// } - -// if abiMap[eventJson.Address] == nil || abiMap[eventJson.Address][topicSelector] == nil { -// continue -// } - -// // Get the ABI string -// contractAbi, err := abi.JSON(strings.NewReader(abiMap[eventJson.Address][topicSelector]["abi"])) -// if err != nil { -// fmt.Println("Error initializing contract ABI: ", err) -// return nil, err -// } - -// // Decode the event data -// decodedArgs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, eventJson.Topics, eventJson.Data) -// if decodeErr != nil { -// fmt.Println("Error decoding event not decoded data: ", eventJson.TransactionHash, decodeErr) -// decodedArgs = map[string]interface{}{ -// "input_raw": eventJson, -// "abi": abiMap[eventJson.Address][topicSelector]["abi"], -// "selector": topicSelector, -// "error": decodeErr, -// } -// label = indexer.SeerCrawlerRawLabel -// } - -// // Convert decodedArgs map to JSON -// labelDataBytes, err := json.Marshal(decodedArgs) -// if err != nil { -// fmt.Println("Error converting decodedArgs to JSON: ", err) -// return nil, err -// } - -// // Convert event to label -// eventLabel := indexer.EventLabel{ -// Label: label, -// LabelName: abiMap[eventJson.Address][topicSelector]["abi_name"], -// LabelType: "event", -// BlockNumber: eventJson.BlockNumber, -// BlockHash: eventJson.BlockHash, -// Address: eventJson.Address, -// OriginAddress: eventJson.OriginAddress, -// TransactionHash: eventJson.TransactionHash, -// LabelData: string(labelDataBytes), // Convert JSON byte slice to string -// BlockTimestamp: eventJson.BlockTimestamp, -// LogIndex: eventJson.LogIndex, -// } - -// labels = append(labels, eventLabel) - -// } - -// return labels, nil -// } - func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { var tx *seer_common.TransactionJson err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + // Get blocks in range blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - return nil, err + return nil, nil, err } // Get transactions in range for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + for _, tx := range block.Transactions { label := indexer.SeerCrawlerLabel @@ -1049,6 +925,8 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer continue } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx // Process transaction labels @@ -1058,13 +936,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) if err != nil { fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, err + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) if err != nil { fmt.Println("Error decoding input data: ", err) - return nil, err + return nil, nil, err } decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) @@ -1083,7 +961,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if err != nil { fmt.Println("Error fetching transaction receipt: ", err) - return nil, err + return nil, nil, err } // check if the transaction was successful @@ -1096,17 +974,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, err - } - - blockNumber, err := strconv.ParseUint(tx.BlockNumber, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) - } - - uintBlockTimestamp, err := strconv.ParseUint(block.Timestamp, 10, 64) - if err != nil { - log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + return nil, nil, err } // Convert transaction to label @@ -1121,7 +989,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa Label: label, TransactionHash: tx.Hash, LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: uintBlockTimestamp, + BlockTimestamp: blockTimestamp, } transactionsLabels = append(transactionsLabels, transactionLabel) @@ -1131,23 +999,28 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } - return transactionsLabels, nil + return transactionsLabels, blocksCache, nil } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + // Get events in range var addresses []common.Address - var topics [][]common.Hash + var topics []common.Hash for address, selectorMap := range abiMap { for selector, _ := range selectorMap { - addresses = append(addresses, common.HexToAddress(address)) - topics = append(topics, []common.Hash{common.HexToHash(selector)}) + topics = append(topics, common.HexToHash(selector)) } + + addresses = append(addresses, common.HexToAddress(address)) } // query filter from abiMap @@ -1155,9 +1028,8 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ FromBlock: big.NewInt(int64(startBlock)), ToBlock: big.NewInt(int64(endBlock)), Addresses: addresses, - Topics: topics, + Topics: [][]common.Hash{topics}, } - logs, err := c.ClientFilterLogs(context.Background(), filter, false) if err != nil { @@ -1208,24 +1080,40 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ return nil, err } - // get transaction information - transaction, err := c.GetTransactionByHash(context.Background(), log.TransactionHash) - + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) if err != nil { return nil, err } - blockNumber, err := strconv.ParseUint(log.BlockNumber, 10, 64) - if err != nil { - return nil, err - } + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } - blockTimestamp, err := strconv.ParseUint(transaction.BlockTimestamp, 10, 64) - if err != nil { - return nil, err } - logIndex, err := strconv.ParseUint(log.LogIndex, 10, 64) + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) if err != nil { return nil, err } @@ -1241,7 +1129,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ OriginAddress: transaction.FromAddress, TransactionHash: log.TransactionHash, LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: blockTimestamp, + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, LogIndex: logIndex, } diff --git a/cmd.go b/cmd.go index 602f85c..d4a5e21 100644 --- a/cmd.go +++ b/cmd.go @@ -906,7 +906,7 @@ func CreateHistoricalSyncCommand() *cobra.Command { return synchonizerErr } - err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, customerIds, addresses, batchSize, auto, threads) + err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, addresses, customerIds, batchSize, auto, threads) if err != nil { return err diff --git a/indexer/db.go b/indexer/db.go index 6d325a6..09e29b9 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -1152,18 +1152,37 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') `) + fmt.Println("Addresses:", addresses) + if len(addresses) > 0 { queryBuilder.WriteString(" AND address = ANY(@addresses) ") - queryArgs["addresses"] = addresses + + // decode addresses + addressesBytes := make([][]byte, len(addresses)) + for i, address := range addresses { + addressBytes, err := decodeAddress(address) + if err != nil { + return nil, nil, err + } + addressesBytes[i] = addressBytes // Assign directly to the index + } + + queryArgs["addresses"] = addressesBytes } + fmt.Println("Customer IDs:", customersIds) + if len(customersIds) > 0 { queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") queryArgs["customer_ids"] = customersIds } + fmt.Println("Query:", queryBuilder.String()) + fmt.Println("Query Args:", queryArgs) + rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) if err != nil { + log.Println("Error querying ABI jobs from database", err) return nil, nil, err } @@ -1200,8 +1219,14 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus "abi_type": abiJob.AbiType, } + if abiJob.DeploymentBlockNumber == nil { + value := uint64(1) + abiJob.DeploymentBlockNumber = &value + } + // Retrieve the struct from the map deployInfo, exists := addressDeployBlockDict[address] + if !exists { // Initialize the struct if it doesn't exist deployInfo = AbiJobsDeployInfo{ diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 20976fa..0f2a25b 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -14,6 +14,7 @@ import ( "time" seer_blockchain "github.com/moonstream-to/seer/blockchain" + "github.com/moonstream-to/seer/blockchain/common" "github.com/moonstream-to/seer/crawler" "github.com/moonstream-to/seer/indexer" "github.com/moonstream-to/seer/storage" @@ -410,6 +411,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s return fmt.Errorf("error getting latest block number: %w", err) } d.startBlock = indexedLatestBlock + fmt.Printf("Start block is %d\n", d.startBlock) } // Automatically update ABI jobs as active if auto mode is enabled @@ -425,9 +427,13 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s return fmt.Errorf("error selecting ABI jobs: %w", err) } + fmt.Printf("Found %d customer updates\n", len(customerUpdates)) + // Filter out blocks more for address, abisInfo := range addressesAbisInfo { + fmt.Printf("Address %s, block %d\n", address, abisInfo.DeployedBlockNumber) if abisInfo.DeployedBlockNumber > d.startBlock { + fmt.Printf("Deleting address %s\n", address, abisInfo.DeployedBlockNumber) delete(addressesAbisInfo, address) } } @@ -485,10 +491,14 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s } if useRPC { - d.endBlock = d.startBlock - batchSize - if d.endBlock == 0 { - d.endBlock = 1 + // Calculate the end block + if d.startBlock > batchSize { + d.endBlock = d.startBlock - batchSize + } else { + d.endBlock = 1 // Prevent underflow by setting endBlock to 1 if startBlock < batchSize } + + fmt.Printf("Start block is %d, end block is %d\n", d.startBlock, d.endBlock) } // Read raw data from storage or via RPC @@ -529,7 +539,11 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s for _, update := range customerUpdates { wg.Add(1) - go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) + if useRPC { + go d.processRPCCustomerUpdate(update, customerDBConnections, sem, errChan, &wg) + } else { + go d.processProtoCustomerUpdate(update, rawData, customerDBConnections, sem, errChan, &wg) + } } wg.Wait() @@ -631,7 +645,8 @@ func (d *Synchronizer) processRPCCustomerUpdate( for address, selectorMap := range update.Abis { for selector, abiInfo := range selectorMap { - if abiInfo["type"] == "event" { + + if abiInfo["abi_type"] == "event" { if _, ok := eventAbis[address]; !ok { eventAbis[address] = make(map[string]map[string]string) } @@ -646,30 +661,50 @@ func (d *Synchronizer) processRPCCustomerUpdate( } - // Read transactions and events from the blockchain - // Transactions - transactions, err := d.Client.GetTransactionsLabels(d.startBlock, d.endBlock, transactionAbis) + var transactions []indexer.TransactionLabel + var blocksCache map[uint64]common.BlockWithTransactions - if err != nil { - errChan <- fmt.Errorf("error getting transactions for customer %s: %w", update.CustomerID, err) - <-sem // Release semaphore - return + if len(transactionAbis) != 0 { + + transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.startBlock, d.endBlock, transactionAbis) + + if err != nil { + log.Println("Error getting transactions for customer", update.CustomerID, ":", err) + errChan <- fmt.Errorf("error getting transactions for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } + } else { + transactions = make([]indexer.TransactionLabel, 0) } + fmt.Printf("Transactions length: %d\n", len(transactions)) + // Events - events, err := d.Client.GetEventsLabels(d.startBlock, d.endBlock, eventAbis) + var events []indexer.EventLabel - if err != nil { + if len(eventAbis) != 0 { - errChan <- fmt.Errorf("error getting events for customer %s: %w", update.CustomerID, err) - <-sem // Release semaphore - return + events, err = d.Client.GetEventsLabels(d.endBlock, d.startBlock, eventAbis, blocksCache) + if err != nil { + + log.Println("Error getting events for customer", update.CustomerID, ":", err) + + errChan <- fmt.Errorf("error getting events for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + + } + } else { + events = make([]indexer.EventLabel, 0) } + fmt.Printf("Events length: %d\n", len(events)) + customer.Pgx.WriteLabes(d.blockchain, transactions, events) <-sem // Release semaphore From 6f7794fcc2ee5443384d5f1f897359b31b9a90ca Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 14 Sep 2024 19:52:10 +0300 Subject: [PATCH 08/17] Temp synchronizer update. --- blockchain/arbitrum_one/arbitrum_one.go | 63 +-- .../arbitrum_sepolia/arbitrum_sepolia.go | 63 +-- blockchain/b3/b3.go | 371 ++++++++++++++++-- blockchain/b3_sepolia/b3_sepolia.go | 312 ++++++++++++++- blockchain/ethereum/ethereum.go | 57 +-- .../game7_orbit_arbitrum_sepolia.go | 63 +-- blockchain/game7_testnet/game7_testnet.go | 63 +-- blockchain/handlers.go | 6 +- blockchain/imx_zkevm/imx_zkevm.go | 57 +-- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 57 +-- blockchain/mantle/mantle.go | 57 +-- blockchain/mantle_sepolia/mantle_sepolia.go | 57 +-- blockchain/polygon/polygon.go | 57 +-- blockchain/sepolia/sepolia.go | 57 +-- blockchain/xai/xai.go | 63 +-- blockchain/xai_sepolia/xai_sepolia.go | 63 +-- cmd.go | 14 +- indexer/db.go | 4 +- synchronizer/synchronizer.go | 4 +- 19 files changed, 1098 insertions(+), 390 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index dbfc662..b7d4dc5 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_one", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumOneBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +480,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.B BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +514,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumOneBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumOneBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumOneBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -816,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -840,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index bf8bac9..eb48acb 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_sepolia", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +480,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +514,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -816,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -840,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/b3/b3.go b/blockchain/b3/b3.go index f2a9390..cd5b14c 100644 --- a/blockchain/b3/b3.go +++ b/blockchain/b3/b3.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -16,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +107,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -214,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -224,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -342,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("b3", @@ -373,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &B3BlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *B3BlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -457,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *B3BlocksBatch) *seer_common.BlocksBatc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -490,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *B3Block { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -612,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*B3Block, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch B3BlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -629,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -669,13 +681,27 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -725,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -781,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -806,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -842,4 +866,273 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil -} \ No newline at end of file +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + var blocksCache map[uint64]seer_common.BlockWithTransactions + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 20) + + if err != nil { + fmt.Println("Error fetching blocks: ", err) + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + fmt.Println("Processing block: ", blockNumber) + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, nil, err + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, blocksCache, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + // Get events in range + + var addresses []common.Address + var topics []common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) + } + + addresses = append(addresses, common.HexToAddress(address)) + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) + if err != nil { + return nil, err + } + + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + + } + + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + +} diff --git a/blockchain/b3_sepolia/b3_sepolia.go b/blockchain/b3_sepolia/b3_sepolia.go index 58f71d2..d8b23f6 100644 --- a/blockchain/b3_sepolia/b3_sepolia.go +++ b/blockchain/b3_sepolia/b3_sepolia.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math/big" + "strconv" "strings" "sync" "time" @@ -16,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" "google.golang.org/protobuf/proto" @@ -71,10 +73,10 @@ func (c *Client) GetLatestBlockNumber() (*big.Int, error) { } // BlockByNumber returns the block with the given number. -func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int) (*seer_common.BlockJson, error) { +func (c *Client) GetBlockByNumber(ctx context.Context, number *big.Int, withTransactions bool) (*seer_common.BlockJson, error) { var rawResponse json.RawMessage // Use RawMessage to capture the entire JSON response - err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), true) + err := c.rpcClient.CallContext(ctx, &rawResponse, "eth_getBlockByNumber", "0x"+number.Text(16), withTransactions) if err != nil { fmt.Println("Error calling eth_getBlockByNumber: ", err) return nil, err @@ -105,6 +107,27 @@ func (c *Client) TransactionReceipt(ctx context.Context, hash common.Hash) (*typ return receipt, err } +// Get bytecode of a contract by address. +func (c *Client) GetCode(ctx context.Context, address common.Address, blockNumber uint64) ([]byte, error) { + var code hexutil.Bytes + if blockNumber == 0 { + latestBlockNumber, err := c.GetLatestBlockNumber() + if err != nil { + return nil, err + } + blockNumber = latestBlockNumber.Uint64() + } + err := c.rpcClient.CallContext(ctx, &code, "eth_getCode", address, "0x"+fmt.Sprintf("%x", blockNumber)) + if err != nil { + log.Printf("Failed to get code for address %s at block %d: %v", address.Hex(), blockNumber, err) + return nil, err + } + + if len(code) == 0 { + return nil, nil + } + return code, nil +} func (c *Client) ClientFilterLogs(ctx context.Context, q ethereum.FilterQuery, debug bool) ([]*seer_common.EventJson, error) { var logs []*seer_common.EventJson fromBlock := q.FromBlock @@ -185,7 +208,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int, debug bool) ([]*seer_comm ctx := context.Background() // For simplicity, using a background context; consider timeouts for production. for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { - block, err := c.GetBlockByNumber(ctx, i) + block, err := c.GetBlockByNumber(ctx, i, true) if err != nil { return nil, err } @@ -224,7 +247,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque sem <- struct{}{} // Acquire semaphore - block, getErr := c.GetBlockByNumber(ctx, b) + block, getErr := c.GetBlockByNumber(ctx, b, true) if getErr != nil { log.Printf("Failed to fetch block number: %d, error: %v", b, getErr) errChan <- getErr @@ -663,7 +686,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) @@ -676,6 +699,20 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma label = indexer.SeerCrawlerRawLabel } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) if err != nil { fmt.Println("Error converting decodedArgsTx to JSON: ", err) @@ -842,4 +879,267 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa } return labels, nil -} \ No newline at end of file +} + +func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_common.TransactionJson, error) { + var tx *seer_common.TransactionJson + err := c.rpcClient.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) + return tx, err +} + +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { + var transactionsLabels []indexer.TransactionLabel + + var blocksCache map[uint64]seer_common.BlockWithTransactions + + // Get blocks in range + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + + if err != nil { + return nil, nil, err + } + + // Get transactions in range + + for _, block := range blocks { + + blockNumber, err := strconv.ParseUint(block.BlockNumber, 0, 64) + if err != nil { + log.Fatalf("Failed to convert BlockNumber to uint64: %v", err) + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + + if err != nil { + log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + + label := indexer.SeerCrawlerLabel + + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue + } + // Fill blocks cache + blocksCache[blockNumber].Transactions[tx.Hash] = tx + + // Process transaction labels + + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI transactions: ", err) + return nil, nil, err + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + fmt.Println("Error decoding input data: ", err) + return nil, nil, err + } + + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + + if err != nil { + fmt.Println("Error fetching transaction receipt: ", err) + return nil, nil, err + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } + + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + fmt.Println("Error converting decodedArgsTx to JSON: ", err) + return nil, nil, err + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: blockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blockTimestamp, + } + + transactionsLabels = append(transactionsLabels, transactionLabel) + } + + } + + } + + return transactionsLabels, blocksCache, nil + +} + +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { + var eventsLabels []indexer.EventLabel + + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + // Get events in range + + var addresses []common.Address + var topics []common.Hash + + for address, selectorMap := range abiMap { + for selector, _ := range selectorMap { + topics = append(topics, common.HexToHash(selector)) + } + + addresses = append(addresses, common.HexToAddress(address)) + } + + // query filter from abiMap + filter := ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(startBlock)), + ToBlock: big.NewInt(int64(endBlock)), + Addresses: addresses, + Topics: [][]common.Hash{topics}, + } + logs, err := c.ClientFilterLogs(context.Background(), filter, false) + + if err != nil { + return nil, err + } + + for _, log := range logs { + var decodedArgsLogs map[string]interface{} + label := indexer.SeerCrawlerLabel + + var topicSelector string + + if len(log.Topics) > 0 { + topicSelector = log.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } + + if abiMap[log.Address] == nil || abiMap[log.Address][topicSelector] == nil { + continue + } + + // Get the ABI string + contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) + if err != nil { + fmt.Println("Error initializing contract ABI: ", err) + return nil, err + } + + // Decode the event data + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": log, + "abi": abiMap[log.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } + + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + fmt.Println("Error converting decodedArgsLogs to JSON: ", err) + return nil, err + } + + blockNumber, err := strconv.ParseUint(log.BlockNumber, 0, 64) + if err != nil { + return nil, err + } + + if _, ok := blocksCache[blockNumber]; !ok { + + // get block from rpc + block, err := c.GetBlockByNumber(context.Background(), big.NewInt(int64(blockNumber)), true) + if err != nil { + return nil, err + } + + blockTimestamp, err := strconv.ParseUint(block.Timestamp, 0, 64) + if err != nil { + return nil, err + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ + BlockNumber: blockNumber, + BlockHash: block.Hash, + BlockTimestamp: blockTimestamp, + Transactions: make(map[string]seer_common.TransactionJson), + } + + for _, tx := range block.Transactions { + blocksCache[blockNumber].Transactions[tx.Hash] = tx + } + + } + + transaction := blocksCache[blockNumber].Transactions[log.TransactionHash] + + logIndex, err := strconv.ParseUint(log.LogIndex, 0, 64) + if err != nil { + return nil, err + } + + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelType: "event", + BlockNumber: blockNumber, + BlockHash: log.BlockHash, + Address: log.Address, + OriginAddress: transaction.FromAddress, + TransactionHash: log.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: blocksCache[blockNumber].BlockTimestamp, + LogIndex: logIndex, + } + + eventsLabels = append(eventsLabels, eventLabel) + + } + + return eventsLabels, nil + +} diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 0d7190c..4fd91b6 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("ethereum", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &EthereumBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *EthereumBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*EthereumBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch EthereumBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index f25885a..64ed657 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_orbit_arbitrum_sepolia", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7OrbitArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +480,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +514,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7OrbitArbitrumSepoliaBl TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7OrbitArbitrumSepoliaB func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7OrbitArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -816,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -840,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 8f98bda..543abfa 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_testnet", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7TestnetBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +480,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common. BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +514,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7TestnetBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7TestnetBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7TestnetBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -816,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -840,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/handlers.go b/blockchain/handlers.go index 6df454c..3dfeb0d 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -14,6 +14,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/moonstream-to/seer/blockchain/arbitrum_one" "github.com/moonstream-to/seer/blockchain/arbitrum_sepolia" + "github.com/moonstream-to/seer/blockchain/b3" + "github.com/moonstream-to/seer/blockchain/b3_sepolia" seer_common "github.com/moonstream-to/seer/blockchain/common" "github.com/moonstream-to/seer/blockchain/ethereum" "github.com/moonstream-to/seer/blockchain/game7_orbit_arbitrum_sepolia" @@ -71,10 +73,10 @@ func NewClient(chain, url string, timeout int) (BlockchainClient, error) { client, err := imx_zkevm_sepolia.NewClient(url, timeout) return client, err } else if chain == "b3" { - client, err := imx_zkevm_sepolia.NewClient(url, timeout) + client, err := b3.NewClient(url, timeout) return client, err } else if chain == "b3_sepolia" { - client, err := imx_zkevm_sepolia.NewClient(url, timeout) + client, err := b3_sepolia.NewClient(url, timeout) return client, err } else { return nil, errors.New("unsupported chain type") diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index feae909..9639a0e 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 9fb847b..ffdffd9 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm_sepolia", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 78cded5..90c5a73 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.Blocks BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index c7c87e8..67426d6 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle_sepolia", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleSepoliaBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 335c83a..831965a 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("polygon", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &PolygonBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *PolygonBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*PolygonBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch PolygonBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index d4c116d..f6d1497 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("sepolia", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &SepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,6 +480,11 @@ func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), + + + + + Transactions: txs, }) } @@ -507,6 +513,11 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *SepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), + + + + + } } @@ -624,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*SepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch SepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -641,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -675,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -751,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -806,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -830,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 903e27e..4e4cac2 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +480,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBat BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +514,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -816,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -840,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 7403f4d..ed2f919 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,6 +365,7 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } + // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai_sepolia", @@ -395,14 +396,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -479,10 +480,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.Bl BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -513,10 +514,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -634,12 +635,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiSepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -651,10 +652,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -685,15 +686,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -761,15 +762,16 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } + // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -816,6 +818,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error + for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -840,9 +843,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/cmd.go b/cmd.go index 1166ae4..fccef61 100644 --- a/cmd.go +++ b/cmd.go @@ -801,6 +801,7 @@ func CreateDatabaseOperationCommand() *cobra.Command { } var jobChain, address, abiFile, customerId, userId string + var deployBlock uint64 createJobsCommand := &cobra.Command{ Use: "create-jobs", @@ -833,14 +834,16 @@ func CreateDatabaseOperationCommand() *cobra.Command { } // detect deploy block + if deployBlock == 0 { + deployBlockFromChain, deployErr := seer_blockchain.FindDeployedBlock(client, address) - deploy_block, deployErr := seer_blockchain.FindDeployedBlock(client, address) - - if deployErr != nil { - return deployErr + if deployErr != nil { + return deployErr + } + deployBlock = deployBlockFromChain } - createJobsErr := indexer.DBConnection.CreateJobsFromAbi(jobChain, address, abiFile, customerId, userId, deploy_block) + createJobsErr := indexer.DBConnection.CreateJobsFromAbi(jobChain, address, abiFile, customerId, userId, deployBlock) if createJobsErr != nil { return createJobsErr } @@ -854,6 +857,7 @@ func CreateDatabaseOperationCommand() *cobra.Command { createJobsCommand.Flags().StringVar(&abiFile, "abi-file", "", "The path to the ABI file") createJobsCommand.Flags().StringVar(&customerId, "customer-id", "", "The customer ID to create jobs for (default: '')") createJobsCommand.Flags().StringVar(&userId, "user-id", "00000000-0000-0000-0000-000000000000", "The user ID to create jobs for (default: '00000000-0000-0000-0000-000000000000')") + createJobsCommand.Flags().Uint64Var(&deployBlock, "deploy-block", 0, "The block number to deploy contract (default: 0)") indexCommand.AddCommand(deploymentBlocksCommand) indexCommand.AddCommand(createJobsCommand) diff --git a/indexer/db.go b/indexer/db.go index edd62fd..e14f29e 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -1472,8 +1472,8 @@ func (p *PostgreSQLpgx) CreateJobsFromAbi(chain string, address string, abiFile log.Println("Error decoding address:", err, address) continue } - - _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) + _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson) + //_, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) if err != nil { return err diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index a0c58bf..760b1b6 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -667,8 +667,8 @@ func (d *Synchronizer) processRPCCustomerUpdate( var blocksCache map[uint64]common.BlockWithTransactions if len(transactionAbis) != 0 { - - transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.startBlock, d.endBlock, transactionAbis) + fmt.Printf("Getting transactions for customer %s\n", update.CustomerID) + transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.endBlock, d.startBlock, transactionAbis) if err != nil { log.Println("Error getting transactions for customer", update.CustomerID, ":", err) From cccaaeed4100c57e436438fe7ceee6792e1fcfe1 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 14 Sep 2024 19:52:46 +0300 Subject: [PATCH 09/17] Apply formating. --- blockchain/arbitrum_one/arbitrum_one.go | 63 +++++++++---------- .../arbitrum_sepolia/arbitrum_sepolia.go | 63 +++++++++---------- blockchain/b3_sepolia/b3_sepolia.go | 57 +++++++---------- blockchain/ethereum/ethereum.go | 57 +++++++---------- .../game7_orbit_arbitrum_sepolia.go | 63 +++++++++---------- blockchain/game7_testnet/game7_testnet.go | 63 +++++++++---------- blockchain/imx_zkevm/imx_zkevm.go | 57 +++++++---------- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 57 +++++++---------- blockchain/mantle/mantle.go | 57 +++++++---------- blockchain/mantle_sepolia/mantle_sepolia.go | 57 +++++++---------- blockchain/polygon/polygon.go | 57 +++++++---------- blockchain/sepolia/sepolia.go | 57 +++++++---------- blockchain/xai/xai.go | 63 +++++++++---------- blockchain/xai_sepolia/xai_sepolia.go | 63 +++++++++---------- 14 files changed, 356 insertions(+), 478 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index b7d4dc5..dbfc662 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_one", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumOneBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumOneBlocksBatch) *seer_common.B BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumOneBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumOneBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumOneBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index eb48acb..bf8bac9 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("arbitrum_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *ArbitrumSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ArbitrumSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ArbitrumSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/b3_sepolia/b3_sepolia.go b/blockchain/b3_sepolia/b3_sepolia.go index d8b23f6..488e674 100644 --- a/blockchain/b3_sepolia/b3_sepolia.go +++ b/blockchain/b3_sepolia/b3_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("b3_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &B3SepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *B3SepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *B3SepoliaBlocksBatch) *seer_common.Blo BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *B3SepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*B3SepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch B3SepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 4fd91b6..0d7190c 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("ethereum", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &EthereumBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *EthereumBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *EthereumBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*EthereumBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch EthereumBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 64ed657..f25885a 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_orbit_arbitrum_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7OrbitArbitrumSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7OrbitArbitrumSepoliaBlocksBatch) BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7OrbitArbitrumSepoliaBl TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7OrbitArbitrumSepoliaB func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7OrbitArbitrumSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 543abfa..8f98bda 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("game7_testnet", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &Game7TestnetBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *Game7TestnetBlocksBatch) *seer_common. BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *Game7TestnetBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*Game7TestnetBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch Game7TestnetBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 9639a0e..feae909 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmBlocksBatch) *seer_common.Bloc BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index ffdffd9..9fb847b 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("imx_zkevm_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &ImxZkevmSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *ImxZkevmSepoliaBlocksBatch) *seer_comm BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *ImxZkevmSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*ImxZkevmSepoliaBlock, erro func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch ImxZkevmSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 90c5a73..78cded5 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleBlocksBatch) *seer_common.Blocks BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index 67426d6..c7c87e8 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("mantle_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &MantleSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *MantleSepoliaBlocksBatch) *seer_common BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *MantleSepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*MantleSepoliaBlock, error) func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch MantleSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 831965a..335c83a 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("polygon", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &PolygonBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *PolygonBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *PolygonBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*PolygonBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch PolygonBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index f6d1497..d4c116d 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &SepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,11 +479,6 @@ func ToEntireBlocksBatchFromLogProto(obj *SepoliaBlocksBatch) *seer_common.Block BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - - - - - Transactions: txs, }) } @@ -513,11 +507,6 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *SepoliaBlock { TotalDifficulty: obj.TotalDifficulty, TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - - - - - } } @@ -635,12 +624,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*SepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch SepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +641,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +675,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +751,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +806,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +830,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 4e4cac2..903e27e 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiBlocksBatch) *seer_common.BlocksBat BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index ed2f919..7403f4d 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -237,7 +237,7 @@ func (c *Client) FetchBlocksInRangeAsync(from, to *big.Int, debug bool, maxReque blockNumbersRange = append(blockNumbersRange, new(big.Int).Set(i)) } - sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency + sem := make(chan struct{}, maxRequests) // Semaphore to control concurrency errChan := make(chan error, 1) for _, b := range blockNumbersRange { @@ -365,7 +365,6 @@ func (c *Client) FetchAsProtoBlocksWithEvents(from, to *big.Int, debug bool, max } } } - // Prepare blocks to index blocksIndex = append(blocksIndex, indexer.NewBlockIndex("xai_sepolia", @@ -396,14 +395,14 @@ func (c *Client) ProcessBlocksToBatch(msgs []proto.Message) (proto.Message, erro } return &XaiSepoliaBlocksBatch{ - Blocks: blocks, + Blocks: blocks, SeerVersion: version.SeerVersion, }, nil } func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.BlocksBatchJson { blocksBatchJson := seer_common.BlocksBatchJson{ - Blocks: []seer_common.BlockJson{}, + Blocks: []seer_common.BlockJson{}, SeerVersion: obj.SeerVersion, } @@ -480,10 +479,10 @@ func ToEntireBlocksBatchFromLogProto(obj *XaiSepoliaBlocksBatch) *seer_common.Bl BaseFeePerGas: b.BaseFeePerGas, IndexedAt: fmt.Sprintf("%d", b.IndexedAt), - MixHash: b.MixHash, - SendCount: b.SendCount, - SendRoot: b.SendRoot, - L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), + MixHash: b.MixHash, + SendCount: b.SendCount, + SendRoot: b.SendRoot, + L1BlockNumber: fmt.Sprintf("%d", b.L1BlockNumber), Transactions: txs, }) @@ -514,10 +513,10 @@ func ToProtoSingleBlock(obj *seer_common.BlockJson) *XaiSepoliaBlock { TransactionsRoot: obj.TransactionsRoot, IndexedAt: fromHex(obj.IndexedAt).Uint64(), - MixHash: obj.MixHash, - SendCount: obj.SendCount, - SendRoot: obj.SendRoot, - L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), + MixHash: obj.MixHash, + SendCount: obj.SendCount, + SendRoot: obj.SendRoot, + L1BlockNumber: fromHex(obj.L1BlockNumber).Uint64(), } } @@ -635,12 +634,12 @@ func (c *Client) DecodeProtoBlocks(data []string) ([]*XaiSepoliaBlock, error) { func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_common.BlocksBatchJson, error) { var protoBlocksBatch XaiSepoliaBlocksBatch - dataBytes := rawData.Bytes() + dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %v", err) + } blocksBatchJson := ToEntireBlocksBatchFromLogProto(&protoBlocksBatch) @@ -652,10 +651,10 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma dataBytes := rawData.Bytes() - err := proto.Unmarshal(dataBytes, &protoBlocksBatch) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) - } + err := proto.Unmarshal(dataBytes, &protoBlocksBatch) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) + } var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel @@ -686,15 +685,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma fmt.Println("Error decoding input data: ", err) return nil, nil, err } - + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[tx.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -762,16 +761,15 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, err } - // Decode the event data decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + "abi": abiMap[e.Address][topicSelector]["abi"], + "selector": topicSelector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } @@ -818,7 +816,6 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa var decodedArgs map[string]interface{} var decodeErr error - for _, transaction := range decodedTransactions { label := indexer.SeerCrawlerLabel @@ -843,9 +840,9 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + "abi": abiMap[transaction.ToAddress][selector]["abi"], + "selector": selector, + "error": decodeErr, } label = indexer.SeerCrawlerRawLabel } From f5280b52b12097f7e856571c5b6d954893ca5748 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 24 Sep 2024 15:04:01 +0300 Subject: [PATCH 10/17] Add p --- blockchain/arbitrum_one/arbitrum_one.go | 337 ++++++++++------- .../arbitrum_sepolia/arbitrum_sepolia.go | 337 ++++++++++------- blockchain/b3/b3.go | 348 ++++++++++-------- blockchain/b3_sepolia/b3_sepolia.go | 337 ++++++++++------- blockchain/blockchain.go.tmpl | 347 ++++++++++------- blockchain/ethereum/ethereum.go | 337 ++++++++++------- .../game7_orbit_arbitrum_sepolia.go | 337 ++++++++++------- blockchain/game7_testnet/game7_testnet.go | 337 ++++++++++------- blockchain/handlers.go | 8 +- blockchain/imx_zkevm/imx_zkevm.go | 337 ++++++++++------- .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 337 ++++++++++------- blockchain/mantle/mantle.go | 337 ++++++++++------- blockchain/mantle_sepolia/mantle_sepolia.go | 337 ++++++++++------- blockchain/polygon/polygon.go | 337 ++++++++++------- blockchain/sepolia/sepolia.go | 337 ++++++++++------- blockchain/xai/xai.go | 337 ++++++++++------- blockchain/xai_sepolia/xai_sepolia.go | 337 ++++++++++------- cmd.go | 10 +- indexer/db.go | 32 +- indexer/types.go | 23 +- synchronizer/synchronizer.go | 45 ++- 21 files changed, 3248 insertions(+), 2283 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index dbfc662..9d7578c 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -646,7 +645,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ArbitrumOneBlocksBatch dataBytes := rawData.Bytes() @@ -656,155 +655,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ArbitrumOneBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -822,10 +875,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -834,13 +889,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -862,7 +917,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -884,7 +939,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -933,10 +988,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -945,12 +1003,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -983,7 +1041,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1003,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1054,19 +1112,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1121,7 +1180,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index bf8bac9..cb44a97 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -646,7 +645,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ArbitrumSepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -656,155 +655,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ArbitrumSepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -822,10 +875,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -834,13 +889,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -862,7 +917,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -884,7 +939,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -933,10 +988,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -945,12 +1003,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -983,7 +1041,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1003,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1054,19 +1112,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1121,7 +1180,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/b3/b3.go b/blockchain/b3/b3.go index cd5b14c..1dbdb8e 100644 --- a/blockchain/b3/b3.go +++ b/blockchain/b3/b3.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch B3BlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,210 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *B3Block) { + defer wg.Done() + defer func() { <-semaphoreChan }() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + localTxLabels = append(localTxLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) } + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +866,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +880,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +908,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,19 +930,15 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel - var blocksCache map[uint64]seer_common.BlockWithTransactions - if blocksCache == nil { - blocksCache = make(map[uint64]seer_common.BlockWithTransactions) - } + var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 20) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) if err != nil { - fmt.Println("Error fetching blocks: ", err) return nil, nil, err } @@ -905,8 +957,6 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } - fmt.Println("Processing block: ", blockNumber) - blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -929,10 +979,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -941,12 +994,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -979,7 +1032,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -999,7 +1052,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1050,19 +1103,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1117,7 +1171,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/b3_sepolia/b3_sepolia.go b/blockchain/b3_sepolia/b3_sepolia.go index 488e674..7685535 100644 --- a/blockchain/b3_sepolia/b3_sepolia.go +++ b/blockchain/b3_sepolia/b3_sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch B3SepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *B3SepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index 80cfc05..d8e413e 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -647,7 +646,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch {{.BlockchainName}}BlocksBatch dataBytes := rawData.Bytes() @@ -657,156 +656,211 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error - for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + var wg sync.WaitGroup - label := indexer.SeerCrawlerLabel + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) - // Process transaction labels - selector := tx.Input[:10] - - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err - } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } - - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, - } - label = indexer.SeerCrawlerRawLabel - } + // Iterate over blocks and launch goroutines + for _, b := range protoBlocksBatch.Blocks { + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *{{.BlockchainName}}Block) { + defer wg.Done() - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + label := indexer.SeerCrawlerLabel - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - txLabels = append(txLabels, transactionLabel) - } + // Process transaction labels + selector := tx.Input[:10] + + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue + } + + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } + + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - var topicSelector string + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } + + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" + localTxLabels = append(localTxLabels, transactionLabel) } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + abiEntryLog := abiMap[e.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue + } + + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) + } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -825,10 +879,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -837,13 +893,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -865,7 +921,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -887,7 +943,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -936,10 +992,14 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -948,12 +1008,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -986,7 +1046,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1006,7 +1066,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1057,19 +1117,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1124,7 +1185,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 0d7190c..2b0521a 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch EthereumBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *EthereumBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index f25885a..c8c449f 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -646,7 +645,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch Game7OrbitArbitrumSepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -656,155 +655,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *Game7OrbitArbitrumSepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -822,10 +875,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -834,13 +889,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -862,7 +917,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -884,7 +939,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -933,10 +988,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -945,12 +1003,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -983,7 +1041,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1003,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1054,19 +1112,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1121,7 +1180,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 8f98bda..8ee76c6 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -646,7 +645,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch Game7TestnetBlocksBatch dataBytes := rawData.Bytes() @@ -656,155 +655,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *Game7TestnetBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -822,10 +875,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -834,13 +889,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -862,7 +917,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -884,7 +939,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -933,10 +988,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -945,12 +1003,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -983,7 +1041,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1003,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1054,19 +1112,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1121,7 +1180,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/handlers.go b/blockchain/handlers.go index 3dfeb0d..c437d1f 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -96,12 +96,12 @@ type BlockchainClient interface { FetchAsProtoBlocksWithEvents(*big.Int, *big.Int, bool, int) ([]proto.Message, []indexer.BlockIndex, uint64, error) ProcessBlocksToBatch([]proto.Message) (proto.Message, error) DecodeProtoEntireBlockToJson(*bytes.Buffer) (*seer_common.BlocksBatchJson, error) - DecodeProtoEntireBlockToLabels(*bytes.Buffer, map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) - DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) + DecodeProtoEntireBlockToLabels(*bytes.Buffer, map[string]map[string]*indexer.AbiEntry, int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) + DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) ChainType() string GetCode(context.Context, common.Address, uint64) ([]byte, error) - GetTransactionsLabels(uint64, uint64, map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) - GetEventsLabels(uint64, uint64, map[string]map[string]map[string]string, map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) + GetTransactionsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) + GetEventsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry, map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) } func GetLatestBlockNumberWithRetry(client BlockchainClient, retryAttempts int, retryWaitTime time.Duration) (*big.Int, error) { diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index feae909..2f79ab2 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ImxZkevmBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ImxZkevmBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 9fb847b..6c70b4e 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch ImxZkevmSepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *ImxZkevmSepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 78cded5..436e647 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch MantleBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *MantleBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index c7c87e8..462b38a 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch MantleSepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *MantleSepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 335c83a..a5d588b 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch PolygonBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *PolygonBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index d4c116d..4af0d4e 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -636,7 +635,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch SepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -646,155 +645,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *SepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -812,10 +865,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -824,13 +879,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -852,7 +907,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -874,7 +929,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -923,10 +978,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -935,12 +993,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -973,7 +1031,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -993,7 +1051,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1044,19 +1102,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1111,7 +1170,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 903e27e..8cf24be 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -646,7 +645,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch XaiBlocksBatch dataBytes := rawData.Bytes() @@ -656,155 +655,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *XaiBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -822,10 +875,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -834,13 +889,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -862,7 +917,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -884,7 +939,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -933,10 +988,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -945,12 +1003,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -983,7 +1041,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1003,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1054,19 +1112,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1121,7 +1180,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 7403f4d..c33e8be 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -646,7 +645,7 @@ func (c *Client) DecodeProtoEntireBlockToJson(rawData *bytes.Buffer) (*seer_comm return blocksBatchJson, nil } -func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]map[string]string) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.EventLabel, []indexer.TransactionLabel, error) { var protoBlocksBatch XaiSepoliaBlocksBatch dataBytes := rawData.Bytes() @@ -656,155 +655,209 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma return nil, nil, fmt.Errorf("failed to unmarshal data: %v", err) } + // Shared slices to collect labels var labels []indexer.EventLabel var txLabels []indexer.TransactionLabel + var labelsMutex sync.Mutex + var decodeErr error + var wg sync.WaitGroup + + // Concurrency limit (e.g., 10 goroutines at a time) + concurrencyLimit := threads + semaphoreChan := make(chan struct{}, concurrencyLimit) + + // Channel to collect errors from goroutines + errorChan := make(chan error, len(protoBlocksBatch.Blocks)) + + // Iterate over blocks and launch goroutines for _, b := range protoBlocksBatch.Blocks { - for _, tx := range b.Transactions { - var decodedArgsTx map[string]interface{} + wg.Add(1) + semaphoreChan <- struct{}{} + go func(b *XaiSepoliaBlock) { + defer wg.Done() - label := indexer.SeerCrawlerLabel + // Local slices to collect labels for this block + var localEventLabels []indexer.EventLabel + var localTxLabels []indexer.TransactionLabel - if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer - continue - } + for _, tx := range b.Transactions { + var decodedArgsTx map[string]interface{} - // Process transaction labels - selector := tx.Input[:10] + label := indexer.SeerCrawlerLabel - if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + if len(tx.Input) < 10 { // If input is less than 3 characters then it direct transfer + continue } - inputData, err := hex.DecodeString(tx.Input[2:]) - if err != nil { - fmt.Println("Error decoding input data: ", err) - return nil, nil, err - } + // Process transaction labels + selector := tx.Input[:10] - decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) - if decodeErr != nil { - fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) - decodedArgsTx = map[string]interface{}{ - "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], - "selector": selector, - "error": decodeErr, + if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { + + txAbiEntry := abiMap[tx.ToAddress][selector] + + var initErr error + txAbiEntry.Once.Do(func() { + txAbiEntry.Abi, initErr = seer_common.GetABI(txAbiEntry.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || txAbiEntry.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for address %s: %v", tx.ToAddress, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + inputData, err := hex.DecodeString(tx.Input[2:]) + if err != nil { + errorChan <- fmt.Errorf("error decoding input data for tx %s: %v", tx.Hash, err) + continue + } + decodedArgsTx, decodeErr = seer_common.DecodeTransactionInputDataToInterface(txAbiEntry.Abi, inputData) + if decodeErr != nil { + fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) + decodedArgsTx = map[string]interface{}{ + "input_raw": tx, + "abi": txAbiEntry.AbiJSON, + "selector": selector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - if err != nil { - fmt.Println("Error fetching transaction receipt: ", err) - return nil, nil, err - } + receipt, err := c.TransactionReceipt(context.Background(), common.HexToHash(tx.Hash)) + if err != nil { + errorChan <- fmt.Errorf("error getting transaction receipt for tx %s: %v", tx.Hash, err) + continue + } - // check if the transaction was successful - if receipt.Status == 1 { - decodedArgsTx["status"] = 1 - } else { - decodedArgsTx["status"] = 0 - } + // check if the transaction was successful + if receipt.Status == 1 { + decodedArgsTx["status"] = 1 + } else { + decodedArgsTx["status"] = 0 + } - txLabelDataBytes, err := json.Marshal(decodedArgsTx) - if err != nil { - fmt.Println("Error converting decodedArgsTx to JSON: ", err) - return nil, nil, err - } + txLabelDataBytes, err := json.Marshal(decodedArgsTx) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsTx to JSON for tx %s: %v", tx.Hash, err) + continue + } - // Convert transaction to label - transactionLabel := indexer.TransactionLabel{ - Address: tx.ToAddress, - BlockNumber: tx.BlockNumber, - BlockHash: tx.BlockHash, - CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], - LabelType: "tx_call", - OriginAddress: tx.FromAddress, - Label: label, - TransactionHash: tx.Hash, - LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, + // Convert transaction to label + transactionLabel := indexer.TransactionLabel{ + Address: tx.ToAddress, + BlockNumber: tx.BlockNumber, + BlockHash: tx.BlockHash, + CallerAddress: tx.FromAddress, + LabelName: txAbiEntry.AbiName, + LabelType: "tx_call", + OriginAddress: tx.FromAddress, + Label: label, + TransactionHash: tx.Hash, + LabelData: string(txLabelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + } + + txLabels = append(txLabels, transactionLabel) } - txLabels = append(txLabels, transactionLabel) - } + // Process events + for _, e := range tx.Logs { + var decodedArgsLogs map[string]interface{} + label = indexer.SeerCrawlerLabel - // Process events - for _, e := range tx.Logs { - var decodedArgsLogs map[string]interface{} - label = indexer.SeerCrawlerLabel + var topicSelector string - var topicSelector string + if len(e.Topics) > 0 { + topicSelector = e.Topics[0] + } else { + // 0x0 is the default topic selector + topicSelector = "0x0" + } - if len(e.Topics) > 0 { - topicSelector = e.Topics[0] - } else { - // 0x0 is the default topic selector - topicSelector = "0x0" - } + if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { + continue + } - if abiMap[e.Address] == nil || abiMap[e.Address][topicSelector] == nil { - continue - } + abiEntryLog := abiMap[e.Address][topicSelector] - // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[e.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, nil, err - } + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) - // Decode the event data - decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(&contractAbi, e.Topics, e.Data) - if decodeErr != nil { - fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) - decodedArgsLogs = map[string]interface{}{ - "input_raw": e, - "abi": abiMap[e.Address][topicSelector]["abi"], - "selector": topicSelector, - "error": decodeErr, + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + errorChan <- fmt.Errorf("error getting ABI for log address %s: %v", e.Address, initErr) + continue } - label = indexer.SeerCrawlerRawLabel - } - // Convert decodedArgsLogs map to JSON - labelDataBytes, err := json.Marshal(decodedArgsLogs) - if err != nil { - fmt.Println("Error converting decodedArgsLogs to JSON: ", err) - return nil, nil, err - } + // Decode the event data + decodedArgsLogs, decodeErr = seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, e.Topics, e.Data) + if decodeErr != nil { + fmt.Println("Error decoding event not decoded data: ", e.TransactionHash, decodeErr) + decodedArgsLogs = map[string]interface{}{ + "input_raw": e, + "abi": abiEntryLog.AbiJSON, + "selector": topicSelector, + "error": decodeErr, + } + label = indexer.SeerCrawlerRawLabel + } - // Convert event to label - eventLabel := indexer.EventLabel{ - Label: label, - LabelName: abiMap[e.Address][topicSelector]["abi_name"], - LabelType: "event", - BlockNumber: e.BlockNumber, - BlockHash: e.BlockHash, - Address: e.Address, - OriginAddress: tx.FromAddress, - TransactionHash: e.TransactionHash, - LabelData: string(labelDataBytes), // Convert JSON byte slice to string - BlockTimestamp: b.Timestamp, - LogIndex: e.LogIndex, - } + // Convert decodedArgsLogs map to JSON + labelDataBytes, err := json.Marshal(decodedArgsLogs) + if err != nil { + errorChan <- fmt.Errorf("error converting decodedArgsLogs to JSON for tx %s: %v", e.TransactionHash, err) + continue + } + // Convert event to label + eventLabel := indexer.EventLabel{ + Label: label, + LabelName: abiEntryLog.AbiName, + LabelType: "event", + BlockNumber: e.BlockNumber, + BlockHash: e.BlockHash, + Address: e.Address, + OriginAddress: tx.FromAddress, + TransactionHash: e.TransactionHash, + LabelData: string(labelDataBytes), // Convert JSON byte slice to string + BlockTimestamp: b.Timestamp, + LogIndex: e.LogIndex, + } - labels = append(labels, eventLabel) + labels = append(labels, eventLabel) + } } - } + + // Append local labels to shared slices under mutex + labelsMutex.Lock() + labels = append(labels, localEventLabels...) + txLabels = append(txLabels, localTxLabels...) + labelsMutex.Unlock() + }(b) + } + // Wait for all block processing goroutines to finish + wg.Wait() + close(errorChan) + + // Collect all errors + var errorMessages []string + for err := range errorChan { + errorMessages = append(errorMessages, err.Error()) + } + + // If any errors occurred, return them + if len(errorMessages) > 0 { + return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } return labels, txLabels, nil } -func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, error) { +func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCache map[uint64]uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) { decodedTransactions, err := c.DecodeProtoTransactions(transactions) @@ -822,10 +875,12 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa selector := transaction.Input[:10] - contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"])) - - if err != nil { - return nil, err + if abiMap[transaction.ToAddress][selector].Abi == nil { + abiMap[transaction.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[transaction.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } inputData, err := hex.DecodeString(transaction.Input[2:]) @@ -834,13 +889,13 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa return nil, err } - decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(&contractAbi, inputData) + decodedArgs, decodeErr = seer_common.DecodeTransactionInputDataToInterface(abiMap[transaction.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", transaction.Hash, decodeErr) decodedArgs = map[string]interface{}{ "input_raw": transaction, - "abi": abiMap[transaction.ToAddress][selector]["abi"], + "abi": abiMap[transaction.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -862,7 +917,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa BlockNumber: transaction.BlockNumber, BlockHash: transaction.BlockHash, CallerAddress: transaction.FromAddress, - LabelName: abiMap[transaction.ToAddress][selector]["abi_name"], + LabelName: abiMap[transaction.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: transaction.FromAddress, Label: label, @@ -884,7 +939,7 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions @@ -933,10 +988,13 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa selector := tx.Input[:10] if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - txContractAbi, err := abi.JSON(strings.NewReader(abiMap[tx.ToAddress][selector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI transactions: ", err) - return nil, nil, err + + if abiMap[tx.ToAddress][selector].Abi == nil { + abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err + } } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -945,12 +1003,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(&txContractAbi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector]["abi"], + "abi": abiMap[tx.ToAddress][selector].AbiJSON, "selector": selector, "error": decodeErr, } @@ -983,7 +1041,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector]["abi_name"], + LabelName: abiMap[tx.ToAddress][selector].AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1003,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa } -func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]map[string]string, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { +func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, blocksCache map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) { var eventsLabels []indexer.EventLabel if blocksCache == nil { @@ -1054,19 +1112,20 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ } // Get the ABI string - contractAbi, err := abi.JSON(strings.NewReader(abiMap[log.Address][topicSelector]["abi"])) - if err != nil { - fmt.Println("Error initializing contract ABI: ", err) - return nil, err + if abiMap[log.Address][topicSelector].Abi == nil { + abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) + if err != nil { + fmt.Println("Error getting ABI: ", err) + return nil, err + } } - // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(&contractAbi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector]["abi"], + "abi": abiMap[log.Address][topicSelector].AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1121,7 +1180,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector]["abi_name"], + LabelName: abiMap[log.Address][topicSelector].AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/cmd.go b/cmd.go index fccef61..5b49d0d 100644 --- a/cmd.go +++ b/cmd.go @@ -295,7 +295,7 @@ func CreateCrawlerCommand() *cobra.Command { func CreateSynchronizerCommand() *cobra.Command { var startBlock, endBlock, batchSize uint64 - var timeout int + var timeout, threads int var chain, baseDir, customerDbUriFlag string synchronizerCmd := &cobra.Command{ @@ -336,7 +336,7 @@ func CreateSynchronizerCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { indexer.InitDBConnection() - newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout) + newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout, threads) if synchonizerErr != nil { return synchonizerErr } @@ -365,6 +365,7 @@ func CreateSynchronizerCommand() *cobra.Command { synchronizerCmd.Flags().IntVar(&timeout, "timeout", 30, "The timeout for the crawler in seconds (default: 30)") synchronizerCmd.Flags().Uint64Var(&batchSize, "batch-size", 100, "The number of blocks to crawl in each batch (default: 100)") synchronizerCmd.Flags().StringVar(&customerDbUriFlag, "customer-db-uri", "", "Set customer database URI for development. This workflow bypass fetching customer IDs and its database URL connection strings from mdb-v3-controller API") + synchronizerCmd.Flags().IntVar(&threads, "threads", 5, "Number of go-routines for concurrent decoding") return synchronizerCmd } @@ -835,6 +836,7 @@ func CreateDatabaseOperationCommand() *cobra.Command { // detect deploy block if deployBlock == 0 { + fmt.Println("Deploy block is not provided, trying to find it from chain") deployBlockFromChain, deployErr := seer_blockchain.FindDeployedBlock(client, address) if deployErr != nil { @@ -907,12 +909,12 @@ func CreateHistoricalSyncCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { indexer.InitDBConnection() - newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout) + newSynchronizer, synchonizerErr := synchronizer.NewSynchronizer(chain, baseDir, startBlock, endBlock, batchSize, timeout, threads) if synchonizerErr != nil { return synchonizerErr } - err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, addresses, customerIds, batchSize, auto, threads) + err := newSynchronizer.HistoricalSyncRef(customerDbUriFlag, addresses, customerIds, batchSize, auto) if err != nil { return err diff --git a/indexer/db.go b/indexer/db.go index e14f29e..ff260b0 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -595,7 +595,7 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome abi_selector, json_build_object( 'abi', - '[' || abi || ']', + ('[' || abi || ']')::jsonb, 'abi_name', abi_name ) @@ -630,7 +630,7 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome return 0, 0, "", nil, err } - var customers []map[string]map[string]map[string]map[string]string + var customers []map[string]map[string]map[string]*AbiEntry var path string var firstBlockNumber, lastBlockNumber uint64 @@ -708,9 +708,8 @@ func (p *PostgreSQLpgx) EnsureCorrectSelectors(blockchain string, WriteToDB bool for _, abiJob := range abiJobs { - // Get the correct selector for the ABI - abiObj, err := abi.JSON(strings.NewReader("[" + abiJob.Abi + "]")) - + // Now you can use abiJSONStr as a string + abiObj, err := abi.JSON(strings.NewReader(abiJob.Abi)) if err != nil { log.Println("Error parsing ABI for ABI job:", abiJob.ID, err) return err @@ -1150,14 +1149,12 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus queryBuilder.WriteString(` SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, - historical_crawl_status, progress, task_pickedup, '[' || abi || ']' as abi, - (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number + historical_crawl_status, progress, moonworm_task_pickedup as task_pickedup, '[' || abi || ']' as abi, + (abi::jsonb)->>'type' AS abiType, created_at, updated_at, 1 as deployment_block_number FROM abi_jobs WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') `) - fmt.Println("Addresses:", addresses) - if len(addresses) > 0 { queryBuilder.WriteString(" AND address = ANY(@addresses) ") @@ -1174,16 +1171,11 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus queryArgs["addresses"] = addressesBytes } - fmt.Println("Customer IDs:", customersIds) - if len(customersIds) > 0 { queryBuilder.WriteString(" AND customer_id = ANY(@customer_ids) ") queryArgs["customer_ids"] = customersIds } - fmt.Println("Query:", queryBuilder.String()) - fmt.Println("Query Args:", queryArgs) - rows, err := conn.Query(context.Background(), queryBuilder.String(), queryArgs) if err != nil { log.Println("Error querying ABI jobs from database", err) @@ -1209,18 +1201,18 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus if _, exists := customerUpdatesDict[abiJob.CustomerID]; !exists { customerUpdatesDict[abiJob.CustomerID] = CustomerUpdates{ CustomerID: abiJob.CustomerID, - Abis: make(map[string]map[string]map[string]string), + Abis: make(map[string]map[string]*AbiEntry), } } if _, exists := customerUpdatesDict[abiJob.CustomerID].Abis[address]; !exists { - customerUpdatesDict[abiJob.CustomerID].Abis[address] = make(map[string]map[string]string) + customerUpdatesDict[abiJob.CustomerID].Abis[address] = make(map[string]*AbiEntry) } - customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = map[string]string{ - "abi": abiJob.Abi, - "abi_name": abiJob.AbiName, - "abi_type": abiJob.AbiType, + customerUpdatesDict[abiJob.CustomerID].Abis[address][abiJob.AbiSelector] = &AbiEntry{ + AbiJSON: abiJob.Abi, + AbiName: abiJob.AbiName, + AbiType: abiJob.AbiType, } if abiJob.DeploymentBlockNumber == nil { diff --git a/indexer/types.go b/indexer/types.go index 305edaa..cd9c1ee 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -1,6 +1,11 @@ package indexer -import "time" +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi" +) // gorm is a Go ORM library for working with databases @@ -69,10 +74,10 @@ type AbiJob struct { } type CustomerUpdates struct { - CustomerID string `json:"customer_id"` - Abis map[string]map[string]map[string]string `json:"abis"` - LastBlock uint64 `json:"last_block"` - Path string `json:"path"` + CustomerID string `json:"customer_id"` + Abis map[string]map[string]*AbiEntry `json:"abis"` + LastBlock uint64 `json:"last_block"` + Path string `json:"path"` } type TaskForTransaction struct { @@ -120,3 +125,11 @@ type AbiJobsDeployInfo struct { DeployedBlockNumber uint64 IDs []string } + +type AbiEntry struct { + AbiJSON string + Abi *abi.ABI + AbiName string + AbiType string + Once sync.Once +} diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 760b1b6..64dd3bd 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -30,10 +30,11 @@ type Synchronizer struct { batchSize uint64 baseDir string basePath string + threads int } // NewSynchronizer creates a new synchronizer instance with the given blockchain handler. -func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize uint64, timeout int) (*Synchronizer, error) { +func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize uint64, timeout int, threads int) (*Synchronizer, error) { var synchronizer Synchronizer basePath := filepath.Join(baseDir, crawler.SeerCrawlerStoragePrefix, "data", blockchain) @@ -51,6 +52,10 @@ func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize log.Printf("Initialized new synchronizer at blockchain: %s, startBlock: %d, endBlock: %d", blockchain, startBlock, endBlock) + if threads <= 0 { + threads = 1 + } + synchronizer = Synchronizer{ Client: client, StorageInstance: storageInstance, @@ -61,6 +66,7 @@ func NewSynchronizer(blockchain, baseDir string, startBlock, endBlock, batchSize batchSize: batchSize, baseDir: baseDir, basePath: basePath, + threads: threads, } return &synchronizer, nil @@ -399,7 +405,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, nil } -func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool, threads int) error { +func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool) error { var useRPC bool var isCycleFinished bool @@ -534,8 +540,8 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Process customer updates in parallel var wg sync.WaitGroup - sem := make(chan struct{}, threads) // Semaphore to control concurrency - errChan := make(chan error, 1) // Buffered channel for error handling + sem := make(chan struct{}, d.threads) // Semaphore to control concurrency + errChan := make(chan error, 1) // Buffered channel for error handling for _, update := range customerUpdates { wg.Add(1) @@ -596,15 +602,20 @@ func (d *Synchronizer) processProtoCustomerUpdate( return } defer conn.Release() - - decodedEvents, decodedTransactions, err := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis) + decodedEvents, decodedTransactions, err := d.Client.DecodeProtoEntireBlockToLabels(&rawData, update.Abis, d.threads) if err != nil { - errChan <- fmt.Errorf("error decoding events for customer %s: %w", update.CustomerID, err) + errChan <- fmt.Errorf("error %s: %w", update.CustomerID, err) <-sem // Release semaphore return } - customer.Pgx.WriteLabes(d.blockchain, decodedTransactions, decodedEvents) + err = customer.Pgx.WriteLabes(d.blockchain, decodedTransactions, decodedEvents) + + if err != nil { + errChan <- fmt.Errorf("error writing labels for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } <-sem // Release semaphore } @@ -639,21 +650,21 @@ func (d *Synchronizer) processRPCCustomerUpdate( // split abis by the type of the task - eventAbis := make(map[string]map[string]map[string]string) - transactionAbis := make(map[string]map[string]map[string]string) + eventAbis := make(map[string]map[string]*indexer.AbiEntry) + transactionAbis := make(map[string]map[string]*indexer.AbiEntry) for address, selectorMap := range update.Abis { for selector, abiInfo := range selectorMap { - if abiInfo["abi_type"] == "event" { + if abiInfo.AbiType == "event" { if _, ok := eventAbis[address]; !ok { - eventAbis[address] = make(map[string]map[string]string) + eventAbis[address] = make(map[string]*indexer.AbiEntry) } eventAbis[address][selector] = abiInfo } else { if _, ok := transactionAbis[address]; !ok { - transactionAbis[address] = make(map[string]map[string]string) + transactionAbis[address] = make(map[string]*indexer.AbiEntry) } transactionAbis[address][selector] = abiInfo } @@ -705,7 +716,13 @@ func (d *Synchronizer) processRPCCustomerUpdate( fmt.Printf("Events length: %d\n", len(events)) - customer.Pgx.WriteLabes(d.blockchain, transactions, events) + err = customer.Pgx.WriteLabes(d.blockchain, transactions, events) + + if err != nil { + errChan <- fmt.Errorf("error writing labels for customer %s: %w", update.CustomerID, err) + <-sem // Release semaphore + return + } <-sem // Release semaphore } From e246ec7b87fc9d82ca7656e5c697592b5a4b94c4 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 24 Sep 2024 17:30:59 +0300 Subject: [PATCH 11/17] Update missing thread safe object. --- blockchain/arbitrum_one/arbitrum_one.go | 5 +++-- blockchain/arbitrum_sepolia/arbitrum_sepolia.go | 5 +++-- blockchain/b3/b3.go | 2 +- blockchain/b3_sepolia/b3_sepolia.go | 5 +++-- blockchain/blockchain.go.tmpl | 3 ++- blockchain/ethereum/ethereum.go | 5 +++-- .../game7_orbit_arbitrum_sepolia.go | 5 +++-- blockchain/game7_testnet/game7_testnet.go | 5 +++-- blockchain/imx_zkevm/imx_zkevm.go | 5 +++-- blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go | 5 +++-- blockchain/mantle/mantle.go | 5 +++-- blockchain/mantle_sepolia/mantle_sepolia.go | 5 +++-- blockchain/polygon/polygon.go | 5 +++-- blockchain/sepolia/sepolia.go | 5 +++-- blockchain/xai/xai.go | 5 +++-- blockchain/xai_sepolia/xai_sepolia.go | 5 +++-- indexer/db.go | 2 +- 17 files changed, 46 insertions(+), 31 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 9d7578c..6d6c600 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -677,6 +677,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *ArbitrumOneBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -760,7 +761,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -828,7 +829,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index cb44a97..dc418da 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -677,6 +677,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *ArbitrumSepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -760,7 +761,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -828,7 +829,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/b3/b3.go b/blockchain/b3/b3.go index 1dbdb8e..3f040a3 100644 --- a/blockchain/b3/b3.go +++ b/blockchain/b3/b3.go @@ -828,7 +828,6 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma labels = append(labels, localEventLabels...) txLabels = append(txLabels, localTxLabels...) labelsMutex.Unlock() - }(b) } // Wait for all block processing goroutines to finish @@ -845,6 +844,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma if len(errorMessages) > 0 { return nil, nil, fmt.Errorf("errors occurred during processing:\n%s", strings.Join(errorMessages, "\n")) } + return labels, txLabels, nil } diff --git a/blockchain/b3_sepolia/b3_sepolia.go b/blockchain/b3_sepolia/b3_sepolia.go index 7685535..bd4b5a0 100644 --- a/blockchain/b3_sepolia/b3_sepolia.go +++ b/blockchain/b3_sepolia/b3_sepolia.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *B3SepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index d8e413e..d3d9ce9 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -679,8 +679,9 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *{{.BlockchainName}}Block) { defer wg.Done() + defer func() { <-semaphoreChan }() - + // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel var localTxLabels []indexer.TransactionLabel diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 2b0521a..3ef6ee7 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *EthereumBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index c8c449f..9542377 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -677,6 +677,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *Game7OrbitArbitrumSepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -760,7 +761,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -828,7 +829,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 8ee76c6..1316f62 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -677,6 +677,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *Game7TestnetBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -760,7 +761,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -828,7 +829,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 2f79ab2..82e1c70 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *ImxZkevmBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 6c70b4e..971f408 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *ImxZkevmSepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index 436e647..a676c95 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *MantleBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index 462b38a..ddf1d21 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *MantleSepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index a5d588b..3db2082 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *PolygonBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index 4af0d4e..c54e5d9 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -667,6 +667,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *SepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -750,7 +751,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -818,7 +819,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 8cf24be..53b94d9 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -677,6 +677,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *XaiBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -760,7 +761,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -828,7 +829,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index c33e8be..26bd73b 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -677,6 +677,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma semaphoreChan <- struct{}{} go func(b *XaiSepoliaBlock) { defer wg.Done() + defer func() { <-semaphoreChan }() // Local slices to collect labels for this block var localEventLabels []indexer.EventLabel @@ -760,7 +761,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma BlockTimestamp: b.Timestamp, } - txLabels = append(txLabels, transactionLabel) + localTxLabels = append(localTxLabels, transactionLabel) } // Process events @@ -828,7 +829,7 @@ func (c *Client) DecodeProtoEntireBlockToLabels(rawData *bytes.Buffer, abiMap ma LogIndex: e.LogIndex, } - labels = append(labels, eventLabel) + localEventLabels = append(localEventLabels, eventLabel) } } diff --git a/indexer/db.go b/indexer/db.go index ff260b0..f088957 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -1464,7 +1464,7 @@ func (p *PostgreSQLpgx) CreateJobsFromAbi(chain string, address string, abiFile log.Println("Error decoding address:", err, address) continue } - _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson) + _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), now()) ON CONFLICT DO NOTHING", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson) //_, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) if err != nil { From 20bb45352a41e2489fa8a3ff241cebdc59fcaf14 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 25 Sep 2024 13:13:04 +0300 Subject: [PATCH 12/17] Add fixes for RPC workflow sync. Add threads and lazy abi loading. --- blockchain/arbitrum_one/arbitrum_one.go | 54 ++++++++++++------ .../arbitrum_sepolia/arbitrum_sepolia.go | 54 ++++++++++++------ blockchain/b3/b3.go | 54 ++++++++++++------ blockchain/b3_sepolia/b3_sepolia.go | 54 ++++++++++++------ blockchain/blockchain.go.tmpl | 55 +++++++++++++------ blockchain/ethereum/ethereum.go | 54 ++++++++++++------ .../game7_orbit_arbitrum_sepolia.go | 54 ++++++++++++------ blockchain/game7_testnet/game7_testnet.go | 54 ++++++++++++------ blockchain/handlers.go | 2 +- blockchain/imx_zkevm/imx_zkevm.go | 54 ++++++++++++------ .../imx_zkevm_sepolia/imx_zkevm_sepolia.go | 54 ++++++++++++------ blockchain/mantle/mantle.go | 54 ++++++++++++------ blockchain/mantle_sepolia/mantle_sepolia.go | 54 ++++++++++++------ blockchain/polygon/polygon.go | 54 ++++++++++++------ blockchain/sepolia/sepolia.go | 54 ++++++++++++------ blockchain/xai/xai.go | 54 ++++++++++++------ blockchain/xai_sepolia/xai_sepolia.go | 54 ++++++++++++------ synchronizer/synchronizer.go | 2 +- 18 files changed, 579 insertions(+), 290 deletions(-) diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go index 6d6c600..5ae4996 100644 --- a/blockchain/arbitrum_one/arbitrum_one.go +++ b/blockchain/arbitrum_one/arbitrum_one.go @@ -940,13 +940,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -967,6 +967,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -990,12 +994,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1004,12 +1017,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1042,7 +1055,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1112,21 +1125,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1181,7 +1199,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go index dc418da..bdfd94f 100644 --- a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go +++ b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go @@ -940,13 +940,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -967,6 +967,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -990,12 +994,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1004,12 +1017,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1042,7 +1055,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1112,21 +1125,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1181,7 +1199,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/b3/b3.go b/blockchain/b3/b3.go index 3f040a3..602a58f 100644 --- a/blockchain/b3/b3.go +++ b/blockchain/b3/b3.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/b3_sepolia/b3_sepolia.go b/blockchain/b3_sepolia/b3_sepolia.go index bd4b5a0..f881c64 100644 --- a/blockchain/b3_sepolia/b3_sepolia.go +++ b/blockchain/b3_sepolia/b3_sepolia.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl index d3d9ce9..bbd5a70 100644 --- a/blockchain/blockchain.go.tmpl +++ b/blockchain/blockchain.go.tmpl @@ -944,13 +944,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -971,6 +971,11 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -995,12 +1000,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1009,12 +1023,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1047,7 +1061,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1117,21 +1131,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1186,7 +1205,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index 3ef6ee7..a9e960c 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go index 9542377..946c2f7 100644 --- a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go +++ b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go @@ -940,13 +940,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -967,6 +967,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -990,12 +994,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1004,12 +1017,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1042,7 +1055,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1112,21 +1125,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1181,7 +1199,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/game7_testnet/game7_testnet.go b/blockchain/game7_testnet/game7_testnet.go index 1316f62..9928402 100644 --- a/blockchain/game7_testnet/game7_testnet.go +++ b/blockchain/game7_testnet/game7_testnet.go @@ -940,13 +940,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -967,6 +967,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -990,12 +994,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1004,12 +1017,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1042,7 +1055,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1112,21 +1125,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1181,7 +1199,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/handlers.go b/blockchain/handlers.go index c437d1f..e7148cb 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -100,7 +100,7 @@ type BlockchainClient interface { DecodeProtoTransactionsToLabels([]string, map[uint64]uint64, map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, error) ChainType() string GetCode(context.Context, common.Address, uint64) ([]byte, error) - GetTransactionsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) + GetTransactionsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry, int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) GetEventsLabels(uint64, uint64, map[string]map[string]*indexer.AbiEntry, map[uint64]seer_common.BlockWithTransactions) ([]indexer.EventLabel, error) } diff --git a/blockchain/imx_zkevm/imx_zkevm.go b/blockchain/imx_zkevm/imx_zkevm.go index 82e1c70..986c600 100644 --- a/blockchain/imx_zkevm/imx_zkevm.go +++ b/blockchain/imx_zkevm/imx_zkevm.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go index 971f408..64ad7ae 100644 --- a/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go +++ b/blockchain/imx_zkevm_sepolia/imx_zkevm_sepolia.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go index a676c95..7add238 100644 --- a/blockchain/mantle/mantle.go +++ b/blockchain/mantle/mantle.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go index ddf1d21..e85f6fa 100644 --- a/blockchain/mantle_sepolia/mantle_sepolia.go +++ b/blockchain/mantle_sepolia/mantle_sepolia.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go index 3db2082..07acc96 100644 --- a/blockchain/polygon/polygon.go +++ b/blockchain/polygon/polygon.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/sepolia/sepolia.go b/blockchain/sepolia/sepolia.go index c54e5d9..097adde 100644 --- a/blockchain/sepolia/sepolia.go +++ b/blockchain/sepolia/sepolia.go @@ -930,13 +930,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -957,6 +957,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -980,12 +984,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -994,12 +1007,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1032,7 +1045,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1102,21 +1115,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1171,7 +1189,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/xai/xai.go b/blockchain/xai/xai.go index 53b94d9..4dbfa92 100644 --- a/blockchain/xai/xai.go +++ b/blockchain/xai/xai.go @@ -940,13 +940,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -967,6 +967,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -990,12 +994,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1004,12 +1017,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1042,7 +1055,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1112,21 +1125,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1181,7 +1199,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/blockchain/xai_sepolia/xai_sepolia.go b/blockchain/xai_sepolia/xai_sepolia.go index 26bd73b..08bf180 100644 --- a/blockchain/xai_sepolia/xai_sepolia.go +++ b/blockchain/xai_sepolia/xai_sepolia.go @@ -940,13 +940,13 @@ func (c *Client) GetTransactionByHash(ctx context.Context, hash string) (*seer_c return tx, err } -func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { +func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMap map[string]map[string]*indexer.AbiEntry, threads int) ([]indexer.TransactionLabel, map[uint64]seer_common.BlockWithTransactions, error) { var transactionsLabels []indexer.TransactionLabel var blocksCache map[uint64]seer_common.BlockWithTransactions // Get blocks in range - blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, 1) + blocks, err := c.FetchBlocksInRangeAsync(big.NewInt(int64(startBlock)), big.NewInt(int64(endBlock)), false, threads) if err != nil { return nil, nil, err @@ -967,6 +967,10 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa log.Fatalf("Failed to convert BlockTimestamp to uint64: %v", err) } + if blocksCache == nil { + blocksCache = make(map[uint64]seer_common.BlockWithTransactions) + } + blocksCache[blockNumber] = seer_common.BlockWithTransactions{ BlockNumber: blockNumber, BlockHash: block.Hash, @@ -990,12 +994,21 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa if abiMap[tx.ToAddress] != nil && abiMap[tx.ToAddress][selector] != nil { - if abiMap[tx.ToAddress][selector].Abi == nil { - abiMap[tx.ToAddress][selector].Abi, err = seer_common.GetABI(abiMap[tx.ToAddress][selector].AbiJSON) + abiEntryTx := abiMap[tx.ToAddress][selector] + + var err error + abiEntryTx.Once.Do(func() { + abiEntryTx.Abi, err = seer_common.GetABI(abiEntryTx.AbiJSON) if err != nil { fmt.Println("Error getting ABI: ", err) - return nil, nil, err + return } + }) + + // Check if an error occurred during ABI parsing + if abiEntryTx.Abi == nil { + fmt.Println("Error getting ABI: ", err) + return nil, nil, err } inputData, err := hex.DecodeString(tx.Input[2:]) @@ -1004,12 +1017,12 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa return nil, nil, err } - decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiMap[tx.ToAddress][selector].Abi, inputData) + decodedArgsTx, decodeErr := seer_common.DecodeTransactionInputDataToInterface(abiEntryTx.Abi, inputData) if decodeErr != nil { fmt.Println("Error decoding transaction not decoded data: ", tx.Hash, decodeErr) decodedArgsTx = map[string]interface{}{ "input_raw": tx, - "abi": abiMap[tx.ToAddress][selector].AbiJSON, + "abi": abiEntryTx.AbiJSON, "selector": selector, "error": decodeErr, } @@ -1042,7 +1055,7 @@ func (c *Client) GetTransactionsLabels(startBlock uint64, endBlock uint64, abiMa BlockNumber: blockNumber, BlockHash: tx.BlockHash, CallerAddress: tx.FromAddress, - LabelName: abiMap[tx.ToAddress][selector].AbiName, + LabelName: abiEntryTx.AbiName, LabelType: "tx_call", OriginAddress: tx.FromAddress, Label: label, @@ -1112,21 +1125,26 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ continue } - // Get the ABI string - if abiMap[log.Address][topicSelector].Abi == nil { - abiMap[log.Address][topicSelector].Abi, err = seer_common.GetABI(abiMap[log.Address][topicSelector].AbiJSON) - if err != nil { - fmt.Println("Error getting ABI: ", err) - return nil, err - } + abiEntryLog := abiMap[log.Address][topicSelector] + + var initErr error + abiEntryLog.Once.Do(func() { + abiEntryLog.Abi, initErr = seer_common.GetABI(abiEntryLog.AbiJSON) + }) + + // Check if an error occurred during ABI parsing + if initErr != nil || abiEntryLog.Abi == nil { + fmt.Println("Error getting ABI: ", initErr) + return nil, initErr } + // Decode the event data - decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiMap[log.Address][topicSelector].Abi, log.Topics, log.Data) + decodedArgsLogs, decodeErr := seer_common.DecodeLogArgsToLabelData(abiEntryLog.Abi, log.Topics, log.Data) if decodeErr != nil { fmt.Println("Error decoding event not decoded data: ", log.TransactionHash, decodeErr) decodedArgsLogs = map[string]interface{}{ "input_raw": log, - "abi": abiMap[log.Address][topicSelector].AbiJSON, + "abi": abiEntryLog.AbiJSON, "selector": topicSelector, "error": decodeErr, } @@ -1181,7 +1199,7 @@ func (c *Client) GetEventsLabels(startBlock uint64, endBlock uint64, abiMap map[ // Convert event to label eventLabel := indexer.EventLabel{ Label: label, - LabelName: abiMap[log.Address][topicSelector].AbiName, + LabelName: abiEntryLog.AbiName, LabelType: "event", BlockNumber: blockNumber, BlockHash: log.BlockHash, diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 64dd3bd..0810296 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -679,7 +679,7 @@ func (d *Synchronizer) processRPCCustomerUpdate( if len(transactionAbis) != 0 { fmt.Printf("Getting transactions for customer %s\n", update.CustomerID) - transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.endBlock, d.startBlock, transactionAbis) + transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.endBlock, d.startBlock, transactionAbis, d.threads) if err != nil { log.Println("Error getting transactions for customer", update.CustomerID, ":", err) From b29c3fe321fd5a6c14a7f48acc8a10f752123ddb Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 26 Sep 2024 15:11:40 +0300 Subject: [PATCH 13/17] Add fixes for deploy block search. --- blockchain/handlers.go | 4 ++-- cmd.go | 4 +++- indexer/db.go | 16 +++++++--------- indexer/types.go | 2 +- synchronizer/synchronizer.go | 29 +++-------------------------- 5 files changed, 16 insertions(+), 39 deletions(-) diff --git a/blockchain/handlers.go b/blockchain/handlers.go index e7148cb..812c575 100644 --- a/blockchain/handlers.go +++ b/blockchain/handlers.go @@ -151,11 +151,11 @@ func DecodeTransactionInputData(contractABI *abi.ABI, data []byte) { fmt.Printf("Method inputs: %v\n", inputsMap) } -func DeployBlocksLookUpAndUpdate() error { +func DeployBlocksLookUpAndUpdate(blockchain string) error { // get all abi jobs without deployed block - chainsAddresses, err := indexer.DBConnection.GetAbiJobsWithoutDeployBlocks() + chainsAddresses, err := indexer.DBConnection.GetAbiJobsWithoutDeployBlocks(blockchain) if err != nil { log.Printf("Failed to get abi jobs without deployed blocks: %v", err) diff --git a/cmd.go b/cmd.go index 5b49d0d..c658762 100644 --- a/cmd.go +++ b/cmd.go @@ -792,7 +792,7 @@ func CreateDatabaseOperationCommand() *cobra.Command { }, RunE: func(cmd *cobra.Command, args []string) error { - deploymentBlocksErr := seer_blockchain.DeployBlocksLookUpAndUpdate() + deploymentBlocksErr := seer_blockchain.DeployBlocksLookUpAndUpdate(chain) if deploymentBlocksErr != nil { return deploymentBlocksErr } @@ -801,6 +801,8 @@ func CreateDatabaseOperationCommand() *cobra.Command { }, } + deploymentBlocksCommand.Flags().StringVar(&chain, "chain", "ethereum", "The blockchain to crawl (default: ethereum)") + var jobChain, address, abiFile, customerId, userId string var deployBlock uint64 diff --git a/indexer/db.go b/indexer/db.go index f088957..e68a03a 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -480,7 +480,7 @@ func (p *PostgreSQLpgx) ReadABIJobs(blockchain string) ([]AbiJob, error) { defer conn.Release() - rows, err := conn.Query(context.Background(), "SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, (abi::jsonb)->>'type' as abiType, created_at, updated_at, deployment_block_number FROM abi_jobs where chain=$1 and (abi::jsonb)->>'type' is not null", blockchain) + rows, err := conn.Query(context.Background(), "SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, '[' || abi || ']' as abi, (abi::jsonb)->>'type' as abiType, created_at, updated_at, deployment_block_number FROM abi_jobs where chain=$1 and (abi::jsonb)->>'type' is not null", blockchain) if err != nil { return nil, err @@ -1149,10 +1149,10 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus queryBuilder.WriteString(` SELECT id, address, user_id, customer_id, abi_selector, chain, abi_name, status, - historical_crawl_status, progress, moonworm_task_pickedup as task_pickedup, '[' || abi || ']' as abi, - (abi::jsonb)->>'type' AS abiType, created_at, updated_at, 1 as deployment_block_number + historical_crawl_status, progress, moonworm_task_pickedup, '[' || abi || ']' as abi, + (abi::jsonb)->>'type' AS abiType, created_at, updated_at, deployment_block_number FROM abi_jobs - WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') + WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') and deployment_block_number is not null `) if len(addresses) > 0 { @@ -1311,7 +1311,7 @@ func (p *PostgreSQLpgx) FindBatchPath(blockchain string, blockNumber uint64) (st } -func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks() (map[string]map[string][]string, error) { +func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks(blockchain string) (map[string]map[string][]string, error) { pool := p.GetPool() conn, err := pool.Acquire(context.Background()) @@ -1324,8 +1324,7 @@ func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks() (map[string]map[string][ /// get all addresses that not have deploy block number - rows, err := conn.Query(context.Background(), "SELECT id, chain, address FROM abi_jobs WHERE deployment_block_number is null") - + rows, err := conn.Query(context.Background(), "SELECT id, chain, address FROM abi_jobs WHERE deployment_block_number is null and chain=$1 and (abi::jsonb)->>'type' in ('function', 'event') and (abi::jsonb)->>'stateMutability'!='view'", blockchain) if err != nil { log.Println("Error querying abi jobs from database", err) return nil, err @@ -1464,8 +1463,7 @@ func (p *PostgreSQLpgx) CreateJobsFromAbi(chain string, address string, abiFile log.Println("Error decoding address:", err, address) continue } - _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), now()) ON CONFLICT DO NOTHING", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson) - //_, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) + _, err = conn.Exec(context.Background(), "INSERT INTO abi_jobs (id, address, user_id, customer_id, abi_selector, chain, abi_name, status, historical_crawl_status, progress, moonworm_task_pickedup, abi, deployment_block_number, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), now())", jobID, addressBytes, userID, customerID, selector, chain, abiJob["name"], "true", "pending", 0, false, abiJobJson, deployBlock) if err != nil { return err diff --git a/indexer/types.go b/indexer/types.go index cd9c1ee..65880c1 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -65,7 +65,7 @@ type AbiJob struct { Status string HistoricalCrawlStatus string Progress int - TaskPickedup bool + MoonwormTaskPickedup bool Abi string AbiType string CreatedAt time.Time diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 0810296..4fa13ed 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -464,6 +464,8 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s for address, abisInfo := range addressesAbisInfo { if abisInfo.DeployedBlockNumber > d.startBlock { + log.Printf("Finished crawling for address %s at block %d\n", address, abisInfo.DeployedBlockNumber) + // update the status of the address for the customer to done err := indexer.DBConnection.UpdateAbisAsDone(abisInfo.IDs) if err != nil { @@ -509,27 +511,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Read raw data from storage or via RPC var rawData bytes.Buffer - if useRPC { - // protoMessage, _, _, err := seer_blockchain.CrawlEntireBlocks(d.Client, big.NewInt(int64(d.endBlock)), big.NewInt(int64(d.startBlock)), false, 5) - // if err != nil { - // return fmt.Errorf("error reading events via RPC: %w", err) - // } - - // blocksBatch, err := d.Client.ProcessBlocksToBatch(protoMessage) - // if err != nil { - // return fmt.Errorf("error processing blocks to batch: %w", err) - // } - - // dataBytes, err := proto.Marshal(blocksBatch) - // if err != nil { - // return fmt.Errorf("error marshaling protoMessage: %w", err) - // } - - // rawData = *bytes.NewBuffer(dataBytes) - - // Read transactions and events from the blockchain - - } else { + if !useRPC { rawData, err = d.StorageInstance.Read(path) if err != nil { return fmt.Errorf("error reading events from storage: %w", err) @@ -678,7 +660,6 @@ func (d *Synchronizer) processRPCCustomerUpdate( var blocksCache map[uint64]common.BlockWithTransactions if len(transactionAbis) != 0 { - fmt.Printf("Getting transactions for customer %s\n", update.CustomerID) transactions, blocksCache, err = d.Client.GetTransactionsLabels(d.endBlock, d.startBlock, transactionAbis, d.threads) if err != nil { @@ -691,8 +672,6 @@ func (d *Synchronizer) processRPCCustomerUpdate( transactions = make([]indexer.TransactionLabel, 0) } - fmt.Printf("Transactions length: %d\n", len(transactions)) - // Events var events []indexer.EventLabel @@ -714,8 +693,6 @@ func (d *Synchronizer) processRPCCustomerUpdate( events = make([]indexer.EventLabel, 0) } - fmt.Printf("Events length: %d\n", len(events)) - err = customer.Pgx.WriteLabes(d.blockchain, transactions, events) if err != nil { From 3093afe19afff643da2a969a9912bd8bb070c0cc Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 26 Sep 2024 19:21:28 +0300 Subject: [PATCH 14/17] Update deploy with timers Add progress update. --- deploy/deploy.bash | 171 +++++++++++++++++- ...storical-synchronizer-arbitrum-one.service | 16 ++ ...historical-synchronizer-arbitrum-one.timer | 9 + ...ical-synchronizer-arbitrum-sepolia.service | 16 ++ ...orical-synchronizer-arbitrum-sepolia.timer | 9 + ...historical-synchronizer-b3-sepolia.service | 16 ++ ...r-historical-synchronizer-b3-sepolia.timer | 9 + .../seer-historical-synchronizer-b3.service | 16 ++ deploy/seer-historical-synchronizer-b3.timer | 9 + ...r-historical-synchronizer-ethereum.service | 16 ++ ...eer-historical-synchronizer-ethereum.timer | 9 + ...torical-synchronizer-game7-testnet.service | 16 ++ ...istorical-synchronizer-game7-testnet.timer | 9 + ...cal-synchronizer-imx-zkevm-sepolia.service | 16 ++ ...rical-synchronizer-imx-zkevm-sepolia.timer | 9 + ...-historical-synchronizer-imx-zkevm.service | 16 ++ ...er-historical-synchronizer-imx-zkevm.timer | 9 + ...orical-synchronizer-mantle-sepolia.service | 16 ++ ...storical-synchronizer-mantle-sepolia.timer | 9 + ...eer-historical-synchronizer-mantle.service | 16 ++ .../seer-historical-synchronizer-mantle.timer | 9 + ...er-historical-synchronizer-polygon.service | 16 ++ ...seer-historical-synchronizer-polygon.timer | 9 + ...er-historical-synchronizer-sepolia.service | 16 ++ ...seer-historical-synchronizer-sepolia.timer | 9 + ...istorical-synchronizer-xai-sepolia.service | 16 ++ ...-historical-synchronizer-xai-sepolia.timer | 9 + .../seer-historical-synchronizer-xai.service | 16 ++ deploy/seer-historical-synchronizer-xai.timer | 9 + indexer/db.go | 59 +++++- synchronizer/synchronizer.go | 32 +++- 31 files changed, 603 insertions(+), 9 deletions(-) create mode 100644 deploy/seer-historical-synchronizer-arbitrum-one.service create mode 100644 deploy/seer-historical-synchronizer-arbitrum-one.timer create mode 100644 deploy/seer-historical-synchronizer-arbitrum-sepolia.service create mode 100644 deploy/seer-historical-synchronizer-arbitrum-sepolia.timer create mode 100644 deploy/seer-historical-synchronizer-b3-sepolia.service create mode 100644 deploy/seer-historical-synchronizer-b3-sepolia.timer create mode 100644 deploy/seer-historical-synchronizer-b3.service create mode 100644 deploy/seer-historical-synchronizer-b3.timer create mode 100644 deploy/seer-historical-synchronizer-ethereum.service create mode 100644 deploy/seer-historical-synchronizer-ethereum.timer create mode 100644 deploy/seer-historical-synchronizer-game7-testnet.service create mode 100644 deploy/seer-historical-synchronizer-game7-testnet.timer create mode 100644 deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service create mode 100644 deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer create mode 100644 deploy/seer-historical-synchronizer-imx-zkevm.service create mode 100644 deploy/seer-historical-synchronizer-imx-zkevm.timer create mode 100644 deploy/seer-historical-synchronizer-mantle-sepolia.service create mode 100644 deploy/seer-historical-synchronizer-mantle-sepolia.timer create mode 100644 deploy/seer-historical-synchronizer-mantle.service create mode 100644 deploy/seer-historical-synchronizer-mantle.timer create mode 100644 deploy/seer-historical-synchronizer-polygon.service create mode 100644 deploy/seer-historical-synchronizer-polygon.timer create mode 100644 deploy/seer-historical-synchronizer-sepolia.service create mode 100644 deploy/seer-historical-synchronizer-sepolia.timer create mode 100644 deploy/seer-historical-synchronizer-xai-sepolia.service create mode 100644 deploy/seer-historical-synchronizer-xai-sepolia.timer create mode 100644 deploy/seer-historical-synchronizer-xai.service create mode 100644 deploy/seer-historical-synchronizer-xai.timer diff --git a/deploy/deploy.bash b/deploy/deploy.bash index f9cf474..97591f6 100755 --- a/deploy/deploy.bash +++ b/deploy/deploy.bash @@ -54,6 +54,37 @@ SEER_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE="seer-synchronizer-imx-zkevm-se SEER_SYNCHRONIZER_B3_SERVICE_FILE="seer-synchronizer-b3.service" SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE="seer-synchronizer-b3-sepolia.service" +# Historical Synchronizer (timers) +SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE="seer-historical-synchronizer-ethereum.service" +SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE="seer-historical-synchronizer-ethereum.timer" +SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE="seer-historical-synchronizer-polygon.service" +SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE="seer-historical-synchronizer-polygon.timer" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE="seer-historical-synchronizer-arbitrum-one.service" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE="seer-historical-synchronizer-arbitrum-one.timer" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-arbitrum-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-arbitrum-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE="seer-historical-synchronizer-game7-testnet.service" +SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE="seer-historical-synchronizer-game7-testnet.timer" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-mantle-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE="seer-historical-synchronizer-mantle.timer" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE="seer-historical-synchronizer-mantle.service" +SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE="seer-historical-synchronizer-mantle.timer" +SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-xai-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-xai-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE="seer-historical-synchronizer-xai.service" +SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE="seer-historical-synchronizer-xai.timer" +SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE="seer-historical-synchronizer-imx-zkevm.service" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE="seer-historical-synchronizer-imx-zkevm.timer" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-imx-zkevm-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-imx-zkevm-sepolia.timer" +SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE="seer-historical-synchronizer-b3.service" +SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE="seer-historical-synchronizer-b3.timer" +SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE="seer-historical-synchronizer-b3-sepolia.service" +SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE="seer-historical-synchronizer-b3-sepolia.timer" + + set -eu if [ ! -d "${SECRETS_DIR}" ]; then @@ -372,4 +403,142 @@ echo -e "${PREFIX_INFO} Replacing existing seer synchronizer for B3 Sepolia bloc chmod 644 "${SCRIPT_DIR}/${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" cp "${SCRIPT_DIR}/${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" \ No newline at end of file +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" + + +# Historical Synchronizers + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Ethereum blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_ETHEREUM_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Polygon blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_POLYGON_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Arbitrum One blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_ONE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Arbitrum Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_ARBITRUM_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Game7 Testnet blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Mantle Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Mantle blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_MANTLE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Xai Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_XAI_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Xai blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_XAI_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Immutable zkEvm blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Immutable zkEvm Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_IMX_ZKEVM_SEPOLIA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for Game7 Testnet blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_GAME7_TESTNET_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for B3 blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_B3_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing seer historical synchronizer for B3 Sepolia blockchain service and timer with: ${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}, ${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}", "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${SEER_HISTORICAL_SYNCHRONIZER_B3_SEPOLIA_TIMER_FILE}" \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-arbitrum-one.service b/deploy/seer-historical-synchronizer-arbitrum-one.service new file mode 100644 index 0000000..bcb2c84 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-one.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for arbitrum one blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain arbitrum_one +SyslogIdentifier=seer-historical-synchronizer-arbitrum-one + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-arbitrum-one.timer b/deploy/seer-historical-synchronizer-arbitrum-one.timer new file mode 100644 index 0000000..ff74d15 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-one.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on Arbitrum One + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-arbitrum-sepolia.service b/deploy/seer-historical-synchronizer-arbitrum-sepolia.service new file mode 100644 index 0000000..7b283e2 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for arbitrum sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain arbitrum_sepolia +SyslogIdentifier=seer-historical-synchronizer-arbitrum-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-arbitrum-sepolia.timer b/deploy/seer-historical-synchronizer-arbitrum-sepolia.timer new file mode 100644 index 0000000..4bf23f1 --- /dev/null +++ b/deploy/seer-historical-synchronizer-arbitrum-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on Arbitrum sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-b3-sepolia.service b/deploy/seer-historical-synchronizer-b3-sepolia.service new file mode 100644 index 0000000..b3d9cb7 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for b3 sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain b3_sepolia +SyslogIdentifier=seer-historical-synchronizer-ab3-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-b3-sepolia.timer b/deploy/seer-historical-synchronizer-b3-sepolia.timer new file mode 100644 index 0000000..793ee70 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on b3 sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-b3.service b/deploy/seer-historical-synchronizer-b3.service new file mode 100644 index 0000000..e4f9313 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for b3 blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain b3 +SyslogIdentifier=seer-historical-synchronizer-b3 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-b3.timer b/deploy/seer-historical-synchronizer-b3.timer new file mode 100644 index 0000000..1b05711 --- /dev/null +++ b/deploy/seer-historical-synchronizer-b3.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on b3 + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-ethereum.service b/deploy/seer-historical-synchronizer-ethereum.service new file mode 100644 index 0000000..accb275 --- /dev/null +++ b/deploy/seer-historical-synchronizer-ethereum.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Ethereum blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain ethereum +SyslogIdentifier=seer-historical-synchronizer-ethereum + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-ethereum.timer b/deploy/seer-historical-synchronizer-ethereum.timer new file mode 100644 index 0000000..e91c854 --- /dev/null +++ b/deploy/seer-historical-synchronizer-ethereum.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on ethereum + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-game7-testnet.service b/deploy/seer-historical-synchronizer-game7-testnet.service new file mode 100644 index 0000000..293766e --- /dev/null +++ b/deploy/seer-historical-synchronizer-game7-testnet.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for game7 testnet blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain game7_testnet +SyslogIdentifier=seer-historical-synchronizer-game7-testnet + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-game7-testnet.timer b/deploy/seer-historical-synchronizer-game7-testnet.timer new file mode 100644 index 0000000..aa19912 --- /dev/null +++ b/deploy/seer-historical-synchronizer-game7-testnet.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on game7 testnet + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service new file mode 100644 index 0000000..b85f72d --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Immutable zkEvm Sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain imx_zkevm_sepolia +SyslogIdentifier=seer-historical-synchronizer-imx-zkevm-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer new file mode 100644 index 0000000..71804c8 --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on imx zkevm sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-imx-zkevm.service b/deploy/seer-historical-synchronizer-imx-zkevm.service new file mode 100644 index 0000000..796d668 --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Immutable zkEvm blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain imx_zkevm +SyslogIdentifier=seer-historical-synchronizer-imx-zkevm + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-imx-zkevm.timer b/deploy/seer-historical-synchronizer-imx-zkevm.timer new file mode 100644 index 0000000..57fc0a7 --- /dev/null +++ b/deploy/seer-historical-synchronizer-imx-zkevm.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on imx zkevm + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-mantle-sepolia.service b/deploy/seer-historical-synchronizer-mantle-sepolia.service new file mode 100644 index 0000000..23eeb48 --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for mantle sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain mantle_sepolia +SyslogIdentifier=seer-historical-synchronizer-mantle-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-mantle-sepolia.timer b/deploy/seer-historical-synchronizer-mantle-sepolia.timer new file mode 100644 index 0000000..277bd9a --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on mantle sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-mantle.service b/deploy/seer-historical-synchronizer-mantle.service new file mode 100644 index 0000000..44d8a69 --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for mantle blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain mantle +SyslogIdentifier=seer-historical-synchronizer-mantle + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-mantle.timer b/deploy/seer-historical-synchronizer-mantle.timer new file mode 100644 index 0000000..45da398 --- /dev/null +++ b/deploy/seer-historical-synchronizer-mantle.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on mantle + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-polygon.service b/deploy/seer-historical-synchronizer-polygon.service new file mode 100644 index 0000000..3561857 --- /dev/null +++ b/deploy/seer-historical-synchronizer-polygon.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Polygon blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain polygon +SyslogIdentifier=seer-historical-synchronizer-polygon + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-polygon.timer b/deploy/seer-historical-synchronizer-polygon.timer new file mode 100644 index 0000000..0653765 --- /dev/null +++ b/deploy/seer-historical-synchronizer-polygon.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on polygon + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-sepolia.service b/deploy/seer-historical-synchronizer-sepolia.service new file mode 100644 index 0000000..96f2cd7 --- /dev/null +++ b/deploy/seer-historical-synchronizer-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for Sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain sepolia +SyslogIdentifier=seer-historical-synchronizer-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-sepolia.timer b/deploy/seer-historical-synchronizer-sepolia.timer new file mode 100644 index 0000000..f45fab4 --- /dev/null +++ b/deploy/seer-historical-synchronizer-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on Sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-xai-sepolia.service b/deploy/seer-historical-synchronizer-xai-sepolia.service new file mode 100644 index 0000000..67a80e0 --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai-sepolia.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for xai sepolia blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain xai_sepolia +SyslogIdentifier=seer-historical-synchronizer-xai-sepolia + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-xai-sepolia.timer b/deploy/seer-historical-synchronizer-xai-sepolia.timer new file mode 100644 index 0000000..ffc1c2c --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai-sepolia.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on xai sepolia + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/seer-historical-synchronizer-xai.service b/deploy/seer-historical-synchronizer-xai.service new file mode 100644 index 0000000..0ee4ddb --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai.service @@ -0,0 +1,16 @@ +[Unit] +Description=Seer historical synchronizer service for xai blockchain +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/seer +EnvironmentFile=/home/ubuntu/seer-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/seer/seer historical-sync --auto --chain xai +SyslogIdentifier=seer-historical-synchronizer-xai + +[Install] +WantedBy=multi-user.target diff --git a/deploy/seer-historical-synchronizer-xai.timer b/deploy/seer-historical-synchronizer-xai.timer new file mode 100644 index 0000000..d5a8709 --- /dev/null +++ b/deploy/seer-historical-synchronizer-xai.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs seer historical synchronizer on xai + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/indexer/db.go b/indexer/db.go index e68a03a..532de57 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -1113,10 +1113,10 @@ func (p *PostgreSQLpgx) UpdateAbiJobsStatus(blockchain string) error { query := ` UPDATE abi_jobs - SET historical_crawl_status = 'pickedup' + SET historical_crawl_status = 'in_progress', moonworm_task_pickedup = true WHERE chain = @chain AND historical_crawl_status = 'pending' - AND status = true + AND status = 'active' AND deployment_block_number IS NOT NULL ` @@ -1132,7 +1132,7 @@ func (p *PostgreSQLpgx) UpdateAbiJobsStatus(blockchain string) error { return nil } -func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, customersIds []string) ([]CustomerUpdates, map[string]AbiJobsDeployInfo, error) { +func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, customersIds []string, onlyNotDobe bool) ([]CustomerUpdates, map[string]AbiJobsDeployInfo, error) { pool := p.GetPool() conn, err := pool.Acquire(context.Background()) @@ -1155,6 +1155,10 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') and deployment_block_number is not null `) + if onlyNotDobe { + queryBuilder.WriteString(" AND historical_crawl_status != 'done' ") + } + if len(addresses) > 0 { queryBuilder.WriteString(" AND address = ANY(@addresses) ") @@ -1257,7 +1261,7 @@ func (p *PostgreSQLpgx) UpdateAbisAsDone(ids []string) error { query := ` UPDATE abi_jobs - SET historical_crawl_status = 'done' + SET historical_crawl_status = 'done', progress = 100 WHERE id = ANY($1) ` @@ -1324,7 +1328,22 @@ func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks(blockchain string) (map[st /// get all addresses that not have deploy block number - rows, err := conn.Query(context.Background(), "SELECT id, chain, address FROM abi_jobs WHERE deployment_block_number is null and chain=$1 and (abi::jsonb)->>'type' in ('function', 'event') and (abi::jsonb)->>'stateMutability'!='view'", blockchain) + rows, err := conn.Query(context.Background(), `SELECT + id, + chain, + address + FROM + abi_jobs + WHERE + deployment_block_number is null + and chain = $1 + and ( + (abi :: jsonb) ->> 'type' = 'event' + or ( + (abi :: jsonb) ->> 'type' = 'function' + and (abi :: jsonb) ->> 'stateMutability' != 'view' + ) + )`, blockchain) if err != nil { log.Println("Error querying abi jobs from database", err) return nil, err @@ -1375,6 +1394,36 @@ func (p *PostgreSQLpgx) GetAbiJobsWithoutDeployBlocks(blockchain string) (map[st return chainsAddresses, nil } +func (p *PostgreSQLpgx) UpdateAbisProgress(ids []string, process int) error { + pool := p.GetPool() + + conn, err := pool.Acquire(context.Background()) + + if err != nil { + return err + } + + defer conn.Release() + + // Transform the ids to a slice of UUIDs + idsUUID := make([]uuid.UUID, len(ids)) + for i, id := range ids { + idsUUID[i], err = uuid.Parse(id) + if err != nil { + return err + } + } + + _, err = conn.Exec(context.Background(), "UPDATE abi_jobs SET progress=$1 WHERE id=ANY($2)", process, idsUUID) + + if err != nil { + return err + } + + return nil + +} + func (p *PostgreSQLpgx) UpdateAbiJobsDeployBlock(blockNumber uint64, ids []string) error { pool := p.GetPool() diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 4fa13ed..e15e661 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -408,6 +408,8 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool) error { var useRPC bool var isCycleFinished bool + var updateDeadline time.Time + var initialStartBlock uint64 // Initialize start block if 0 if d.startBlock == 0 { @@ -428,7 +430,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s } // Retrieve customer updates and deployment blocks - customerUpdates, addressesAbisInfo, err := indexer.DBConnection.SelectAbiJobs(d.blockchain, addresses, customerIds) + customerUpdates, addressesAbisInfo, err := indexer.DBConnection.SelectAbiJobs(d.blockchain, addresses, customerIds, auto) if err != nil { return fmt.Errorf("error selecting ABI jobs: %w", err) } @@ -437,9 +439,9 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Filter out blocks more for address, abisInfo := range addressesAbisInfo { - fmt.Printf("Address %s, block %d\n", address, abisInfo.DeployedBlockNumber) + log.Printf("Address %s has deployed block %d\n", address, abisInfo.DeployedBlockNumber) if abisInfo.DeployedBlockNumber > d.startBlock { - fmt.Printf("Deleting address %s\n", address, abisInfo.DeployedBlockNumber) + log.Printf("Finished crawling for address %s at block %d\n", address, abisInfo.DeployedBlockNumber) delete(addressesAbisInfo, address) } } @@ -459,6 +461,8 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s return fmt.Errorf("error getting customers: %w", err) } + updateDeadline = time.Now() + // Main processing loop for { for address, abisInfo := range addressesAbisInfo { @@ -475,6 +479,28 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // drop the address delete(addressesAbisInfo, address) } + + if auto { + // Check if the deadline for the update has passed + if updateDeadline.Add(1 * time.Minute).Before(time.Now()) { + fmt.Println("Update proggres for the addresses") + for _, abisInfo := range addressesAbisInfo { + ids := abisInfo.IDs + + // calculate progress as a percentage between 0 and 100 + + progress := 100 - int(100*(d.startBlock-abisInfo.DeployedBlockNumber)/(d.startBlock-initialStartBlock)) + + err := indexer.DBConnection.UpdateAbisProgress(ids, progress) + if err != nil { + continue + } + + } + updateDeadline = time.Now() + } + } + } if len(addressesAbisInfo) == 0 { From b50fc0853f30bac9b2f2c358ae54acdf567a1b82 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 28 Sep 2024 14:24:51 +0300 Subject: [PATCH 15/17] Add changes. --- synchronizer/synchronizer.go | 41 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index e15e661..3e9d9da 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -405,7 +405,7 @@ func (d *Synchronizer) SyncCycle(customerDbUriFlag string) (bool, error) { return isEnd, nil } -func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, auto bool) error { +func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []string, customerIds []string, batchSize uint64, autoJobs bool) error { var useRPC bool var isCycleFinished bool var updateDeadline time.Time @@ -423,14 +423,14 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s } // Automatically update ABI jobs as active if auto mode is enabled - if auto { + if autoJobs { if err := indexer.DBConnection.UpdateAbiJobsStatus(d.blockchain); err != nil { return fmt.Errorf("error updating ABI: %w", err) } } // Retrieve customer updates and deployment blocks - customerUpdates, addressesAbisInfo, err := indexer.DBConnection.SelectAbiJobs(d.blockchain, addresses, customerIds, auto) + customerUpdates, addressesAbisInfo, err := indexer.DBConnection.SelectAbiJobs(d.blockchain, addresses, customerIds, autoJobs) if err != nil { return fmt.Errorf("error selecting ABI jobs: %w", err) } @@ -438,6 +438,7 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s fmt.Printf("Found %d customer updates\n", len(customerUpdates)) // Filter out blocks more + // TODO: Maybe autoJobs only for address, abisInfo := range addressesAbisInfo { log.Printf("Address %s has deployed block %d\n", address, abisInfo.DeployedBlockNumber) if abisInfo.DeployedBlockNumber > d.startBlock { @@ -465,26 +466,27 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s // Main processing loop for { - for address, abisInfo := range addressesAbisInfo { - if abisInfo.DeployedBlockNumber > d.startBlock { - log.Printf("Finished crawling for address %s at block %d\n", address, abisInfo.DeployedBlockNumber) + if autoJobs { - // update the status of the address for the customer to done - err := indexer.DBConnection.UpdateAbisAsDone(abisInfo.IDs) - if err != nil { - return err - } + for address, abisInfo := range addressesAbisInfo { + if abisInfo.DeployedBlockNumber > d.startBlock { - // drop the address - delete(addressesAbisInfo, address) - } + log.Printf("Finished crawling for address %s at block %d\n", address, abisInfo.DeployedBlockNumber) + + // update the status of the address for the customer to done + err := indexer.DBConnection.UpdateAbisAsDone(abisInfo.IDs) + if err != nil { + return err + } + + // drop the address + delete(addressesAbisInfo, address) + } - if auto { // Check if the deadline for the update has passed if updateDeadline.Add(1 * time.Minute).Before(time.Now()) { - fmt.Println("Update proggres for the addresses") - for _, abisInfo := range addressesAbisInfo { + for address, abisInfo := range addressesAbisInfo { ids := abisInfo.IDs // calculate progress as a percentage between 0 and 100 @@ -496,8 +498,11 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s continue } + log.Printf("Updated progress for address %s to %d%%\n", address, progress) + } updateDeadline = time.Now() + } } @@ -518,6 +523,8 @@ func (d *Synchronizer) HistoricalSyncRef(customerDbUriFlag string, addresses []s } if path == "" { + log.Printf("No batch path found for block %d\n", d.startBlock) + log.Println("Switching to RPC mode") useRPC = true } else { d.endBlock = firstBlockOfChunk From 3524f8a3b898f1ab0dc0edcedb9e7ad6c830e664 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 30 Sep 2024 16:59:36 +0300 Subject: [PATCH 16/17] Apdate types mappings. --- indexer/db.go | 11 +++++------ indexer/types.go | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/indexer/db.go b/indexer/db.go index 532de57..9acdcc6 100644 --- a/indexer/db.go +++ b/indexer/db.go @@ -594,10 +594,9 @@ func (p *PostgreSQLpgx) ReadUpdates(blockchain string, fromBlock uint64, custome json_object_agg( abi_selector, json_build_object( - 'abi', - ('[' || abi || ']')::jsonb, - 'abi_name', - abi_name + 'abi', '[' || abi || ']', + 'abi_name', abi_name, + 'abi_type', abi_type ) ) AS abis_per_address FROM @@ -1132,7 +1131,7 @@ func (p *PostgreSQLpgx) UpdateAbiJobsStatus(blockchain string) error { return nil } -func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, customersIds []string, onlyNotDobe bool) ([]CustomerUpdates, map[string]AbiJobsDeployInfo, error) { +func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, customersIds []string, autoJobs bool) ([]CustomerUpdates, map[string]AbiJobsDeployInfo, error) { pool := p.GetPool() conn, err := pool.Acquire(context.Background()) @@ -1155,7 +1154,7 @@ func (p *PostgreSQLpgx) SelectAbiJobs(blockchain string, addresses []string, cus WHERE chain = @chain AND ((abi::jsonb)->>'type' = 'function' or (abi::jsonb)->>'type' = 'event') and deployment_block_number is not null `) - if onlyNotDobe { + if autoJobs { queryBuilder.WriteString(" AND historical_crawl_status != 'done' ") } diff --git a/indexer/types.go b/indexer/types.go index 65880c1..0c730c1 100644 --- a/indexer/types.go +++ b/indexer/types.go @@ -127,9 +127,9 @@ type AbiJobsDeployInfo struct { } type AbiEntry struct { - AbiJSON string + AbiJSON string `json:"abi"` Abi *abi.ABI - AbiName string - AbiType string + AbiName string `json:"abi_name"` + AbiType string `json:"abi_type"` Once sync.Once } From 39c1ad4de9b3660b50d2e9a0f5bd3f2ec958ea43 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 30 Sep 2024 17:22:26 +0300 Subject: [PATCH 17/17] Bump version. --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 6b597fe..6144365 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -var SeerVersion string = "0.1.21" +var SeerVersion string = "0.1.22"