Fix Read big files from GCS

Fix selector cutting for events. Fix join. Fix address formating.
G7DAO · Jun 6, 2024 · 69bcc08 · 69bcc08
1 parent 7e5bda1
commit 69bcc08
Show file tree

Hide file tree

Showing 14 changed files with 303 additions and 310 deletions.
diff --git a/blockchain/arbitrum_one/arbitrum_one.go b/blockchain/arbitrum_one/arbitrum_one.go
@@ -495,15 +495,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -529,7 +527,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -561,10 +559,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -597,7 +592,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/arbitrum_sepolia/arbitrum_sepolia.go b/blockchain/arbitrum_sepolia/arbitrum_sepolia.go
@@ -495,15 +495,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -529,7 +527,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -561,10 +559,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -597,7 +592,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/blockchain.go.tmpl b/blockchain/blockchain.go.tmpl
@@ -495,15 +495,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -529,7 +527,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -561,10 +559,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -597,7 +592,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/ethereum/ethereum.go b/blockchain/ethereum/ethereum.go
@@ -490,15 +490,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -524,7 +522,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -556,10 +554,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -592,7 +587,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go b/blockchain/game7_orbit_arbitrum_sepolia/game7_orbit_arbitrum_sepolia.go
@@ -495,15 +495,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -529,7 +527,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -561,10 +559,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -597,7 +592,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/mantle/mantle.go b/blockchain/mantle/mantle.go
@@ -490,15 +490,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -524,7 +522,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -556,10 +554,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -592,7 +587,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/mantle_sepolia/mantle_sepolia.go b/blockchain/mantle_sepolia/mantle_sepolia.go
@@ -490,15 +490,13 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -524,7 +522,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -557,9 +555,8 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 		selector := transaction.Input[:10]
 
 		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
 
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -592,7 +589,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,

diff --git a/blockchain/polygon/polygon.go b/blockchain/polygon/polygon.go
@@ -480,6 +480,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 	decodedEvents, err := c.DecodeProtoEventLogs(events)
 
 	if err != nil {
+		fmt.Println("Error decoding events: ", err)
 		return nil, err
 	}
 
@@ -490,15 +491,14 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		var topicSelector string
 
 		if len(event.Topics) > 0 {
-			topicSelector = event.Topics[0][:10]
+			topicSelector = event.Topics[0]
 		} else {
 			continue
 		}
 
-		checksumAddress := common.HexToAddress(event.Address).Hex()
-
 		// Get the ABI string
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][topicSelector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[event.Address][topicSelector]["abi"]))
+
 		if err != nil {
 			fmt.Println("Error initializing contract ABI: ", err)
 			return nil, err
@@ -524,7 +524,7 @@ func (c *Client) DecodeProtoEventsToLabels(events []string, blocksCache map[uint
 		// Convert event to label
 		eventLabel := indexer.EventLabel{
 			Label:           indexer.SeerCrawlerLabel,
-			LabelName:       abiMap[checksumAddress][topicSelector]["abi_name"],
+			LabelName:       abiMap[event.Address][topicSelector]["abi_name"],
 			LabelType:       "event",
 			BlockNumber:     event.BlockNumber,
 			BlockHash:       event.BlockHash,
@@ -556,10 +556,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 
 		selector := transaction.Input[:10]
 
-		// To checksum address
-		checksumAddress := common.HexToAddress(transaction.ToAddress).Hex()
-
-		contractAbi, err := abi.JSON(strings.NewReader(abiMap[checksumAddress][selector]["abi"]))
+		contractAbi, err := abi.JSON(strings.NewReader(abiMap[transaction.ToAddress][selector]["abi"]))
 
 		if err != nil {
 			return nil, err
@@ -592,7 +589,7 @@ func (c *Client) DecodeProtoTransactionsToLabels(transactions []string, blocksCa
 			BlockNumber:     transaction.BlockNumber,
 			BlockHash:       transaction.BlockHash,
 			CallerAddress:   transaction.FromAddress,
-			LabelName:       abiMap[checksumAddress][selector]["abi_name"],
+			LabelName:       abiMap[transaction.ToAddress][selector]["abi_name"],
 			LabelType:       "tx_call",
 			OriginAddress:   transaction.FromAddress,
 			Label:           indexer.SeerCrawlerLabel,