From 45ed244e47552212dde114c01a327c2e76573725 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Sat, 23 Nov 2024 09:38:59 -0500 Subject: [PATCH] Add non-voting nodes to cluster This allows users to add non-voting nodes to cluster at cluster start or join time by setting a boolean flag. Signed-off-by: Alexander Scheel --- cmd/devbao/cluster_join.go | 15 +++++++++++++-- cmd/devbao/cluster_start.go | 30 +++++++++++++++++++++++++++--- pkg/bao/cluster.go | 1 + pkg/bao/node.go | 3 ++- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/cmd/devbao/cluster_join.go b/cmd/devbao/cluster_join.go index 8805a8e..21fe67f 100644 --- a/cmd/devbao/cluster_join.go +++ b/cmd/devbao/cluster_join.go @@ -16,9 +16,16 @@ func BuildClusterJoinCommand() *cli.Command { Usage: "join a given node to the cluster", Action: RunClusterJoinCommand, - } - c.Flags = append(c.Flags, ClusterFlags()...) + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "non-voter", + Aliases: []string{"nv"}, + Value: false, + Usage: "mark new node as a non-voter", + }, + }, + } return c } @@ -31,6 +38,8 @@ func RunClusterJoinCommand(cCtx *cli.Context) error { clusterName := cCtx.Args().First() nodeName := cCtx.Args().Get(1) + nonVoter := cCtx.Bool("non-voter") + cluster, err := bao.LoadCluster(clusterName) if err != nil { return fmt.Errorf("error loading cluster: %w", err) @@ -45,6 +54,8 @@ func RunClusterJoinCommand(cCtx *cli.Context) error { return fmt.Errorf("refusing to add node already in a cluster (`%v`)", node.Cluster) } + node.NonVoter = nonVoter + if err := cluster.JoinNodeHACluster(node); err != nil { return fmt.Errorf("failed to join node to cluster: %w", err) } diff --git a/cmd/devbao/cluster_start.go b/cmd/devbao/cluster_start.go index 5e8ce02..4d5379d 100644 --- a/cmd/devbao/cluster_start.go +++ b/cmd/devbao/cluster_start.go @@ -29,6 +29,11 @@ func ClusterInfoFlags() []cli.Flag { Value: 3, Usage: "number of nodes to run in the cluster; suggested to use an odd number.", }, + &cli.IntFlag{ + Name: "non-voter", + Value: 0, + Usage: "number of non-voter nodes to run in the cluster; less than count", + }, &cli.StringFlag{ Name: "listen", Value: "0.0.0.0", @@ -85,11 +90,16 @@ func RunClusterStartCommand(cCtx *cli.Context) error { listen := cCtx.String("listen") + nonVoter := cCtx.Int("non-voter") + if nonVoter < 0 { + return fmt.Errorf("expected at least zero non-voter nodes; got %v", nonVoter) + } + count := cCtx.Int("count") if count < 1 { return fmt.Errorf("required to have at least one node in the cluster; got %v", count) - } else if (count % 2) == 0 { - fmt.Fprintf(os.Stderr, "[warning] it is suggested to have an odd number of nodes in the HA cluster") + } else if ((count - nonVoter) % 2) == 0 { + fmt.Fprintf(os.Stderr, "[warning] it is suggested to have an odd number of voting nodes in the HA cluster") } clusterName := cCtx.Args().First() @@ -110,7 +120,14 @@ func RunClusterStartCommand(cCtx *cli.Context) error { // Build nodes var nodes []*bao.Node for index := 0; index < count; index++ { - name := fmt.Sprintf("%v-node-%d", clusterName, index) + nonVoter := index >= (count - nonVoter) + + nvSuffix := "" + if nonVoter { + nvSuffix = "-nv" + } + + name := fmt.Sprintf("%v-node-%d%v", clusterName, index, nvSuffix) fmt.Printf("starting %v...\n", name) port := portBase + index*100 @@ -159,6 +176,13 @@ func RunClusterStartCommand(cCtx *cli.Context) error { return fmt.Errorf("failed to build node %v: %w", name, err) } + node.NonVoter = nonVoter + if nonVoter { + if err := node.SaveConfig(); err != nil { + return fmt.Errorf("failed to save non-voter config for node %v: %w", name, err) + } + } + if err := node.Start(); err != nil { return fmt.Errorf("failed to start node %v: %w", name, err) } diff --git a/pkg/bao/cluster.go b/pkg/bao/cluster.go index 886af5e..c3c51e9 100644 --- a/pkg/bao/cluster.go +++ b/pkg/bao/cluster.go @@ -286,6 +286,7 @@ func (c *Cluster) JoinNodeHACluster(node *Node) error { resp, err := nodeClient.Sys().RaftJoin(&api.RaftJoinRequest{ LeaderAPIAddr: leaderClient.Address(), Retry: true, + NonVoter: node.NonVoter, }) if err != nil { return fmt.Errorf("failed joining node %v to cluster %v / leader %v: %w", node.Name, c.Name, leaderNode.Name, err) diff --git a/pkg/bao/node.go b/pkg/bao/node.go index e099315..cfb88bf 100644 --- a/pkg/bao/node.go +++ b/pkg/bao/node.go @@ -30,7 +30,8 @@ type Node struct { Token string `json:"token"` UnsealKeys []string `json:"unseal_keys,omitempty"` - Cluster string `json:"cluster,omitempty"` + Cluster string `json:"cluster,omitempty"` + NonVoter bool `json:"non_voter"` } func (n *Node) FromInterface(iface map[string]interface{}) error {