diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go index de2f5ad..1902952 100644 --- a/blockchain/ethereum/ethereum.go +++ b/blockchain/ethereum/ethereum.go @@ -41,9 +41,18 @@ func (c *Client) Close() { // GetLatestBlockNumber returns the latest block number. func (c *Client) GetLatestBlockNumber() (*big.Int, error) { - var blockNumber *big.Int - err := c.rpcClient.CallContext(context.Background(), &blockNumber, "eth_blockNumber") - return blockNumber, err + var result string + if err := c.rpcClient.CallContext(context.Background(), &result, "eth_blockNumber"); err != nil { + return nil, err + } + + // Convert the hex string to *big.Int + blockNumber, ok := new(big.Int).SetString(result, 0) // The 0 base lets the function infer the base from the string prefix. + if !ok { + return nil, fmt.Errorf("invalid block number format: %s", result) + } + + return blockNumber, nil } // BlockByNumber returns the block with the given number. @@ -105,6 +114,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int) ([]*BlockJson, error) { for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) { block, err := c.GetBlockByNumber(ctx, i) + fmt.Println("Block number: ", i) if err != nil { return nil, err } diff --git a/cmd.go b/cmd.go index 95324fd..a1f59e5 100644 --- a/cmd.go +++ b/cmd.go @@ -128,7 +128,9 @@ func CreateCrawlerCommand() *cobra.Command { Short: "Generate crawlers for various blockchains", Run: func(cmd *cobra.Command, args []string) { - crawler := crawler.NewCrawler("http://localhost:8545") + // read the blockchain url from $INFURA_URL + // if it is not set, use the default url + crawler := crawler.NewCrawler("ethereum") crawler.Start() diff --git a/crawler/crawler.go b/crawler/crawler.go index ac2a715..5fb3bd5 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -2,6 +2,7 @@ package crawler import ( "encoding/json" + "fmt" "log" "math/big" "os" @@ -77,8 +78,13 @@ func writeIndicesToFile(indices []interface{}, filePath string) { // small cheat func (c *Crawler) Start() { chainType := "ethereum" + + url := os.Getenv("INFURA_URL") + + fmt.Println("Using blockchain url:", url) + log.Println("Starting crawler...") - client, err := common.NewClient(chainType, "http://localhost:8545") + client, err := common.NewClient(chainType, url) if err != nil { log.Fatal(err) @@ -97,6 +103,8 @@ func (c *Crawler) Start() { log.Fatalf("Failed to get latest block number: %v", err) } + fmt.Println("Latest block number:", latestBlockNumber) + // Assuming latestBlockNumber is *big.Int and operations for big.Int startBlock := new(big.Int).Sub(latestBlockNumber, big.NewInt(100)) // Start 100 blocks behind the latest crawlBatchSize := big.NewInt(10) // Crawl in batches of 10