Add infura and logs.

blockchain/ethereum/ethereum.go

@@ -41,9 +41,18 @@ func (c *Client) Close() {
 
 // GetLatestBlockNumber returns the latest block number.
 func (c *Client) GetLatestBlockNumber() (*big.Int, error) {
-	var blockNumber *big.Int
-	err := c.rpcClient.CallContext(context.Background(), &blockNumber, "eth_blockNumber")
-	return blockNumber, err
+	var result string
+	if err := c.rpcClient.CallContext(context.Background(), &result, "eth_blockNumber"); err != nil {
+		return nil, err
+	}
+
+	// Convert the hex string to *big.Int
+	blockNumber, ok := new(big.Int).SetString(result, 0) // The 0 base lets the function infer the base from the string prefix.
+	if !ok {
+		return nil, fmt.Errorf("invalid block number format: %s", result)
+	}
+
+	return blockNumber, nil
 }
 
 // BlockByNumber returns the block with the given number.
@@ -105,6 +114,7 @@ func (c *Client) FetchBlocksInRange(from, to *big.Int) ([]*BlockJson, error) {
 
 	for i := new(big.Int).Set(from); i.Cmp(to) <= 0; i.Add(i, big.NewInt(1)) {
 		block, err := c.GetBlockByNumber(ctx, i)
+		fmt.Println("Block number: ", i)
 		if err != nil {
 			return nil, err
 		}

cmd.go

@@ -128,7 +128,9 @@ func CreateCrawlerCommand() *cobra.Command {
 		Short: "Generate crawlers for various blockchains",
 		Run: func(cmd *cobra.Command, args []string) {
 
-			crawler := crawler.NewCrawler("http://localhost:8545")
+			// read the blockchain url from $INFURA_URL
+			// if it is not set, use the default url
+			crawler := crawler.NewCrawler("ethereum")
 
 			crawler.Start()
 

crawler/crawler.go

@@ -2,6 +2,7 @@ package crawler
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
 	"math/big"
 	"os"
@@ -77,8 +78,13 @@ func writeIndicesToFile(indices []interface{}, filePath string) { // small cheat
 func (c *Crawler) Start() {
 
 	chainType := "ethereum"
+
+	url := os.Getenv("INFURA_URL")
+
+	fmt.Println("Using blockchain url:", url)
+
 	log.Println("Starting crawler...")
-	client, err := common.NewClient(chainType, "http://localhost:8545")
+	client, err := common.NewClient(chainType, url)
 
 	if err != nil {
 		log.Fatal(err)
@@ -97,6 +103,8 @@ func (c *Crawler) Start() {
 		log.Fatalf("Failed to get latest block number: %v", err)
 	}
 
+	fmt.Println("Latest block number:", latestBlockNumber)
+
 	// Assuming latestBlockNumber is *big.Int and operations for big.Int
 	startBlock := new(big.Int).Sub(latestBlockNumber, big.NewInt(100)) // Start 100 blocks behind the latest
 	crawlBatchSize := big.NewInt(10)                                   // Crawl in batches of 10