Add metrics to Libp2p networking processes

.github/workflows/branch.yml

@@ -0,0 +1,21 @@
+name: Patch Release Branch Workflow
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - 'v?[0-9]+.[0-9]+'
+jobs:
+  call-common-workflow:
+    uses: dominant-strategies/quai-cicd/.github/workflows/deploy-sandbox-common.yml@main
+    with:
+      needs_build: true
+      build_command: "make go-quai"
+      needs_docker: false
+      include_chart: false
+      cloud_deploy: false
+      skip_deploy: true
+      update_version: false
+    secrets:
+      GH_PAT: ${{ secrets.GH_PAT }}
+      GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY2 }}
+      GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }}

.github/workflows/cut-minor-release.yml

@@ -17,6 +17,7 @@ jobs:
       include_chart: false
       cloud_deploy: false
       skip_deploy: true
+      update_version: true
     secrets:
       GH_PAT: ${{ secrets.GH_PAT }}
       GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY2 }}

.github/workflows/dev.yml

@@ -14,7 +14,6 @@ jobs:
       cloud_deploy: false
       skip_deploy: true
       update_version: false
-
     secrets:
       GH_PAT: ${{ secrets.GH_PAT }}
       GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY2 }}

.github/workflows/patch.yml

@@ -1,9 +1,5 @@
 name: Patch Release Branch Workflow
-on:
-  pull_request:
-    types: [closed]
-    branches:
-      - 'v?[0-9]+.[0-9]+'
+on: workflow_dispatch
 jobs:
   call-common-workflow:
     uses: dominant-strategies/quai-cicd/.github/workflows/deploy-sandbox-common.yml@main
@@ -14,6 +10,7 @@ jobs:
       include_chart: false
       cloud_deploy: false
       skip_deploy: true
+      update_version: true
     secrets:
       GH_PAT: ${{ secrets.GH_PAT }}
       GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY2 }}

.gitignore

@@ -1,7 +1,10 @@
 build/bin/
 nodelogs/
+traces/
 /data/
 */config.toml
 */private.key
 .vscode/
-.idea*
+.idea*
+*.env
+*.sh

Makefile

@@ -39,4 +39,4 @@ debug:
 go-quai:
 	$(GORUN) build/ci.go build ./cmd/go-quai
 	@echo "Done building."
-	@echo "Run \"$(GOBIN)/go-quai\" to launch go-quai."
+	@echo "Run \"$(GOBIN)/go-quai\" to launch go-quai."

cmd/go-quai/start.go

@@ -74,22 +74,31 @@ func runStart(cmd *cobra.Command, args []string) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	if viper.IsSet(utils.PprofFlag.Name) {
+		utils.EnablePprof()
+	}
+
 	// create a new p2p node
 	node, err := node.NewNode(ctx)
 	if err != nil {
 		log.Global.WithField("error", err).Fatal("error creating node")
 	}
 
 	logLevel := cmd.Flag(utils.LogLevelFlag.Name).Value.String()
-	// create instance of consensus backend
-	var nodeWG sync.WaitGroup
-	consensus, err := utils.StartQuaiBackend(ctx, node, logLevel, &nodeWG)
+
+	var startingExpansionNumber uint64
+	if viper.IsSet(utils.StartingExpansionNumberFlag.Name) {
+		startingExpansionNumber = viper.GetUint64(utils.StartingExpansionNumberFlag.Name)
+	}
+	// Start the  hierarchical co-ordinator
+	var nodeWg sync.WaitGroup
+	hc := utils.NewHierarchicalCoordinator(node, logLevel, &nodeWg, startingExpansionNumber)
+	err = hc.StartHierarchicalCoordinator()
 	if err != nil {
-		log.Global.WithField("error", err).Fatal("error creating consensus backend")
+		log.Global.WithField("error", err).Fatal("error starting hierarchical coordinator")
 	}
 
 	// start the p2p node
-	node.SetConsensusBackend(consensus)
 	if err := node.Start(); err != nil {
 		log.Global.WithField("error", err).Fatal("error starting node")
 	}
@@ -106,7 +115,8 @@ func runStart(cmd *cobra.Command, args []string) error {
 	<-ch
 	log.Global.Warn("Received 'stop' signal, shutting down gracefully...")
 	cancel()
-	nodeWG.Wait()
+	// stop the hierarchical co-ordinator
+	hc.Stop()
 	if err := node.Stop(); err != nil {
 		panic(err)
 	}

cmd/utils/cmd.go

@@ -1,18 +1,17 @@
 package utils
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
 	"runtime"
 	"slices"
 	"strings"
-	"sync"
 
 	"github.com/spf13/viper"
 
 	"github.com/dominant-strategies/go-quai/common"
+	"github.com/dominant-strategies/go-quai/core/types"
 	"github.com/dominant-strategies/go-quai/internal/quaiapi"
 	"github.com/dominant-strategies/go-quai/log"
 	"github.com/dominant-strategies/go-quai/metrics_config"
@@ -21,87 +20,55 @@ import (
 	"github.com/dominant-strategies/go-quai/quai"
 	"github.com/dominant-strategies/go-quai/quai/quaiconfig"
 	"github.com/dominant-strategies/go-quai/quaistats"
+	"github.com/syndtr/goleveldb/leveldb"
 )
 
-// Create a new instance of the QuaiBackend consensus service
-func StartQuaiBackend(ctx context.Context, p2p quai.NetworkingAPI, logLevel string, nodeWG *sync.WaitGroup) (*quai.QuaiBackend, error) {
-	quaiBackend, _ := quai.NewQuaiBackend()
-	startNode := func(logPath string, location common.Location, slicesRunning []common.Location) {
-		nodeWG.Add(1)
-		go func() {
-			defer nodeWG.Done()
-			logger := log.NewLogger(logPath, logLevel)
-			logger.Info("Starting Node at location", "location", location)
-			stack, apiBackend := makeFullNode(p2p, location, slicesRunning, logger)
-			quaiBackend.SetApiBackend(apiBackend, location)
-			StartNode(stack)
-			// Create a channel to signal when stack.Wait() is done
-			done := make(chan struct{})
-			go func() {
-				stack.Wait()
-				close(done)
-			}()
-
-			select {
-			case <-done:
-				logger.Info("Node stopped normally")
-				stack.Close()
-				return
-			case <-ctx.Done():
-				logger.Info("Context cancelled, shutting down node")
-				stack.Close()
-				return
-			}
-		}()
+func OpenBackendDB() (*leveldb.DB, error) {
+	dataDir := viper.GetString(DataDirFlag.Name)
+	if _, err := os.Stat(dataDir); os.IsNotExist(err) {
+		err := os.MkdirAll(dataDir, 0755)
+		if err != nil {
+			log.Global.Errorf("error creating data directory: %s", err)
+			return nil, err
+		}
 	}
+	dbPath := dataDir + "quaibackend"
 
-	// Set the p2p backend inside the quaiBackend
-	quaiBackend.SetP2PApiBackend(p2p)
-
-	slicesRunning := getSlicesRunning()
-	regionsRunning := getRegionsRunning(slicesRunning)
-
-	// Start nodes in separate goroutines
-	startNode("nodelogs/prime.log", nil, slicesRunning)
-	for _, region := range regionsRunning {
-		nodelogsFileName := "nodelogs/region-" + fmt.Sprintf("%d", region) + ".log"
-		startNode(nodelogsFileName, common.Location{region}, slicesRunning)
-	}
-	for _, slice := range slicesRunning {
-		nodelogsFileName := "nodelogs/zone-" + fmt.Sprintf("%d", slice[0]) + "-" + fmt.Sprintf("%d", slice[1]) + ".log"
-		startNode(nodelogsFileName, slice, slicesRunning)
+	db, err := leveldb.OpenFile(dbPath, nil)
+	if err != nil {
+		return nil, err
 	}
-
-	return quaiBackend, nil
+	return db, err
 }
 
-// setSlicesRunning sets the slices running flag
-func getSlicesRunning() []common.Location {
+// GetRunningZones returns the slices that are processing state (which are only zones)
+func GetRunningZones() []common.Location {
 	slices := strings.Split(viper.GetString(SlicesRunningFlag.Name), ",")
 
 	// Sanity checks
 	if slices[0] == "" {
 		Fatalf("no slices are specified")
 	}
-	if len(slices) > common.NumRegionsInPrime*common.NumZonesInRegion {
-		Fatalf("number of slices exceed the current ontology")
-	}
-	slicesRunning := []common.Location{}
+	runningSlices := []common.Location{}
 	for _, slice := range slices {
-		slicesRunning = append(slicesRunning, common.Location{slice[1] - 48, slice[3] - 48})
+		location := common.Location{slice[1] - 48, slice[3] - 48}
+		if location.Region() > common.MaxRegions || location.Zone() > common.MaxZones {
+			Fatalf("invalid slice: %s", location)
+		}
+		runningSlices = append(runningSlices, location)
 	}
-	return slicesRunning
+	return runningSlices
 }
 
 // getRegionsRunning returns the regions running
-func getRegionsRunning(slicesRunning []common.Location) []byte {
-	regionsRunning := []byte{}
-	for _, slice := range slicesRunning {
-		if !slices.Contains(regionsRunning, slice[0]) {
-			regionsRunning = append(regionsRunning, slice[0])
+func GetRunningRegions(runningSlices []common.Location) []byte {
+	runningRegions := []byte{}
+	for _, slice := range runningSlices {
+		if !slices.Contains(runningRegions, slice[0]) {
+			runningRegions = append(runningRegions, slice[0])
 		}
 	}
-	return regionsRunning
+	return runningRegions
 }
 
 func StartNode(stack *node.Node) {
@@ -112,7 +79,7 @@ func StartNode(stack *node.Node) {
 }
 
 // makeConfigNode loads quai configuration and creates a blank node instance.
-func makeConfigNode(slicesRunning []common.Location, nodeLocation common.Location, logger *log.Logger) (*node.Node, quaiconfig.QuaiConfig) {
+func makeConfigNode(slicesRunning []common.Location, nodeLocation common.Location, currentExpansionNumber uint8, logger *log.Logger) (*node.Node, quaiconfig.QuaiConfig) {
 	// Load defaults.
 	cfg := quaiconfig.QuaiConfig{
 		Quai:    quaiconfig.Defaults,
@@ -130,7 +97,7 @@ func makeConfigNode(slicesRunning []common.Location, nodeLocation common.Locatio
 	if err != nil {
 		Fatalf("Failed to create the protocol stack: %v", err)
 	}
-	SetQuaiConfig(stack, &cfg.Quai, slicesRunning, nodeLocation, logger)
+	SetQuaiConfig(stack, &cfg.Quai, slicesRunning, nodeLocation, currentExpansionNumber, logger)
 
 	// TODO: Apply stats
 	if viper.IsSet(QuaiStatsURLFlag.Name) {
@@ -150,9 +117,10 @@ func defaultNodeConfig() node.Config {
 }
 
 // makeFullNode loads quai configuration and creates the Quai backend.
-func makeFullNode(p2p quai.NetworkingAPI, nodeLocation common.Location, slicesRunning []common.Location, logger *log.Logger) (*node.Node, quaiapi.Backend) {
-	stack, cfg := makeConfigNode(slicesRunning, nodeLocation, logger)
-	backend, _ := RegisterQuaiService(stack, p2p, cfg.Quai, cfg.Node.NodeLocation.Context(), logger)
+func makeFullNode(p2p quai.NetworkingAPI, nodeLocation common.Location, slicesRunning []common.Location, currentExpansionNumber uint8, genesisBlock *types.WorkObject, logger *log.Logger) (*node.Node, quaiapi.Backend) {
+	stack, cfg := makeConfigNode(slicesRunning, nodeLocation, currentExpansionNumber, logger)
+	startingExpansionNumber := viper.GetUint64(StartingExpansionNumberFlag.Name)
+	backend, _ := RegisterQuaiService(stack, p2p, cfg.Quai, cfg.Node.NodeLocation.Context(), currentExpansionNumber, startingExpansionNumber, genesisBlock, logger)
 	sendfullstats := viper.GetBool(SendFullStatsFlag.Name)
 	// Add the Quai Stats daemon if requested.
 	if cfg.Quaistats.URL != "" {
@@ -164,8 +132,8 @@ func makeFullNode(p2p quai.NetworkingAPI, nodeLocation common.Location, slicesRu
 // RegisterQuaiService adds a Quai client to the stack.
 // The second return value is the full node instance, which may be nil if the
 // node is running as a light client.
-func RegisterQuaiService(stack *node.Node, p2p quai.NetworkingAPI, cfg quaiconfig.Config, nodeCtx int, logger *log.Logger) (quaiapi.Backend, error) {
-	backend, err := quai.New(stack, p2p, &cfg, nodeCtx, logger)
+func RegisterQuaiService(stack *node.Node, p2p quai.NetworkingAPI, cfg quaiconfig.Config, nodeCtx int, currentExpansionNumber uint8, startingExpansionNumber uint64, genesisBlock *types.WorkObject, logger *log.Logger) (quaiapi.Backend, error) {
+	backend, err := quai.New(stack, p2p, &cfg, nodeCtx, currentExpansionNumber, startingExpansionNumber, genesisBlock, logger)
 	if err != nil {
 		Fatalf("Failed to register the Quai service: %v", err)
 	}