horcrux remote signer grpc (#7)

* Add ability to communicate with horcrux via grpc * update horcrux * Allow it to work without kube * 1 second retry * Update docs
strangelove-ventures · Nov 17, 2023 · 3c2446c · 3c2446c
1 parent 7c1fefd
commit 3c2446c
Show file tree

Hide file tree

Showing 7 changed files with 1,162 additions and 161 deletions.
diff --git a/README.md b/README.md
@@ -32,15 +32,33 @@ Additionally, horcrux-proxy will watch the kubernetes cluster for [cosmos-operat
                +
 ```
 
+## Flags
+
+- `-g`/`--grpc-addr` - address to connect to horcrux via GRPC (preferred over listen addresses since grpc allows multiplexing on a single connection)
+- `-l`/`--listen-addr` - add listen address(es) to listen for connection from a horcrux cosigner. If using multiple, it should be to the same cosigner for redundancy. This is deprecated. Use `--grpc-addr` instead.
+- `-o`/`--operator` - when true (default), horcrux-proxy will assume it is running in the same kubernetes cluster as sentries deployed with the [cosmos-operator](https://github.com/strangelove-ventures/cosmos-operator). It will use the kube API to discover operator deployments of `type: Sentry` and automatically connect to them.
+- `-s`/`--sentry` - sentry(ies) to connect to persistently. If using the [cosmos-operator](https://github.com/strangelove-ventures/cosmos-operator), this is likely not necessary.
+- `-a`/`-all` - connect to all sentries regardless of node, instead of only sentries on this node
+
+
 ## Quick Start
 
-Start horcrux-proxy
+If using the [cosmos-operator](https://github.com/strangelove-ventures/cosmos-operator), the required configuration is minimal.
+
+Start command for horcrux-proxy to connect to cosmos operator sentries on the same node:
+
+```bash
+horcrux-proxy start -g $HORCRUX_GRPC_ADDR
+```
+
+Start command for horcrux-proxy to connect to cosmos operator sentries on all nodes:
 
 ```bash
-horcrux-proxy start
+horcrux-proxy start -g $HORCRUX_GRPC_ADDR -a
 ```
 
-### Flags
+Start command for horcrux-proxy to connect to sentries that are not deployed using cosmos-operator:
 
-- `-l`/`--listen-addr` - modify listen address (default `tcp://0.0.0.0:1234`)
-- `-a`/`-all` - connect to all sentries regardless of node, instead of only sentries on this node
+```bash
+horcrux-proxy start -o=false -g $HORCRUX_GRPC_ADDR -s $SENTRY_1 -s $SENTRY_2 ...
+```
diff --git a/cmd/start.go b/cmd/start.go
@@ -8,12 +8,16 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/strangelove-ventures/horcrux-proxy/privval"
+	"github.com/strangelove-ventures/horcrux-proxy/signer"
 )
 
 const (
-	flagLogLevel = "log-level"
-	flagListen   = "listen"
-	flagAll      = "all"
+	flagLogLevel    = "log-level"
+	flagListen      = "listen"
+	flagAll         = "all"
+	flagGRPCAddress = "grpc"
+	flagOperator    = "operator"
+	flagSentry      = "sentry"
 )
 
 func startCmd() *cobra.Command {
@@ -42,15 +46,32 @@ func startCmd() *cobra.Command {
 				listeners[i] = privval.NewSignerListener(logger, addr)
 			}
 
-			loadBalancer := privval.NewRemoteSignerLoadBalancer(logger, listeners)
-			if err = loadBalancer.Start(); err != nil {
-				return fmt.Errorf("failed to start listener(s): %w", err)
+			var hc signer.HorcruxConnection
+
+			grpcAddr, _ := cmd.Flags().GetString(flagGRPCAddress)
+
+			if grpcAddr != "" {
+				hc, err = signer.NewHorcruxGRPCClient(logger, grpcAddr)
+				if err != nil {
+					return fmt.Errorf("failed to create grpc connection: %w", err)
+				}
+			} else {
+				loadBalancer := privval.NewRemoteSignerLoadBalancer(logger, listeners)
+				if err = loadBalancer.Start(); err != nil {
+					return fmt.Errorf("failed to start listener(s): %w", err)
+				}
+				defer logIfErr(logger, loadBalancer.Stop)
+
+				hc = loadBalancer
 			}
-			defer logIfErr(logger, loadBalancer.Stop)
 
 			ctx := cmd.Context()
 
-			watcher, err := NewSentryWatcher(ctx, logger, all, loadBalancer)
+			// if we're running in kubernetes, we can auto-discover sentries
+			operator, _ := cmd.Flags().GetBool(flagOperator)
+			sentries, _ := cmd.Flags().GetStringArray(flagSentry)
+
+			watcher, err := NewSentryWatcher(ctx, logger, all, hc, operator, sentries)
 			if err != nil {
 				return err
 			}
@@ -63,7 +84,10 @@ func startCmd() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringArrayP(flagListen, "l", []string{"tcp://0.0.0.0:1234"}, "Privval listen addresses for the proxy")
+	cmd.Flags().StringArrayP(flagListen, "l", nil, "Privval listen addresses for the proxy (e.g. tcp://0.0.0.0:1234)")
+	cmd.Flags().StringArrayP(flagSentry, "s", nil, "Privval connect addresses for the proxy")
+	cmd.Flags().BoolP(flagOperator, "o", true, "Use this when running in kubernetes with the Cosmos Operator to auto-discover sentries")
+	cmd.Flags().StringP(flagGRPCAddress, "g", "", "GRPC address for the proxy")
 	cmd.Flags().BoolP(flagAll, "a", false, "Connect to sentries on all nodes")
 	cmd.Flags().String(flagLogLevel, "info", "Set log level (debug, info, error, none)")
 

diff --git a/cmd/watcher.go b/cmd/watcher.go
@@ -9,7 +9,6 @@ import (
 	"time"
 
 	cometlog "github.com/cometbft/cometbft/libs/log"
-	"github.com/strangelove-ventures/horcrux-proxy/privval"
 	"github.com/strangelove-ventures/horcrux-proxy/signer"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,12 +23,14 @@ const (
 )
 
 type SentryWatcher struct {
-	all      bool
-	client   *kubernetes.Clientset
-	lb       *privval.RemoteSignerLoadBalancer
-	log      cometlog.Logger
-	node     string
-	sentries map[string]*signer.ReconnRemoteSigner
+	all                bool
+	client             *kubernetes.Clientset
+	hc                 signer.HorcruxConnection
+	log                cometlog.Logger
+	node               string
+	operator           bool
+	persistentSentries []*signer.ReconnRemoteSigner
+	sentries           map[string]*signer.ReconnRemoteSigner
 
 	stop chan struct{}
 	done chan struct{}
@@ -39,51 +40,73 @@ func NewSentryWatcher(
 	ctx context.Context,
 	logger cometlog.Logger,
 	all bool, // should we connect to sentries on all nodes, or just this node?
-	lb *privval.RemoteSignerLoadBalancer,
+	hc signer.HorcruxConnection,
+	operator bool,
+	sentries []string,
 ) (*SentryWatcher, error) {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get in cluster config: %w", err)
-	}
-	// creates the clientset
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create kube clientset: %w", err)
-	}
-
+	var clientset *kubernetes.Clientset
 	var thisNode string
-	if !all {
-		// need to determine which node this pod is on so we can only connect to sentries on this node
 
-		nsbz, err := os.ReadFile(namespaceFile)
+	if operator {
+		config, err := rest.InClusterConfig()
 		if err != nil {
-			return nil, fmt.Errorf("failed to read namespace from service account: %w", err)
+			return nil, fmt.Errorf("failed to get in cluster config: %w", err)
 		}
-		ns := string(nsbz)
-
-		thisPod, err := clientset.CoreV1().Pods(ns).Get(ctx, os.Getenv("HOSTNAME"), metav1.GetOptions{})
+		// creates the clientset
+		clientset, err := kubernetes.NewForConfig(config)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get this pod: %w", err)
+			return nil, fmt.Errorf("failed to create kube clientset: %w", err)
+		}
+
+		if !all {
+			// need to determine which node this pod is on so we can only connect to sentries on this node
+
+			nsbz, err := os.ReadFile(namespaceFile)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read namespace from service account: %w", err)
+			}
+			ns := string(nsbz)
+
+			thisPod, err := clientset.CoreV1().Pods(ns).Get(ctx, os.Getenv("HOSTNAME"), metav1.GetOptions{})
+			if err != nil {
+				return nil, fmt.Errorf("failed to get this pod: %w", err)
+			}
+
+			thisNode = thisPod.Spec.NodeName
 		}
+	}
 
-		thisNode = thisPod.Spec.NodeName
+	persistentSentries := make([]*signer.ReconnRemoteSigner, len(sentries))
+	for i, sentry := range sentries {
+		dialer := net.Dialer{Timeout: 2 * time.Second}
+		persistentSentries[i] = signer.NewReconnRemoteSigner(sentry, logger, hc, dialer)
 	}
 
 	return &SentryWatcher{
-		all:      all,
-		client:   clientset,
-		done:     make(chan struct{}),
-		lb:       lb,
-		log:      logger,
-		node:     thisNode,
-		sentries: make(map[string]*signer.ReconnRemoteSigner),
-		stop:     make(chan struct{}),
+		all:                all,
+		client:             clientset,
+		done:               make(chan struct{}),
+		hc:                 hc,
+		log:                logger,
+		node:               thisNode,
+		operator:           operator,
+		persistentSentries: persistentSentries,
+		sentries:           make(map[string]*signer.ReconnRemoteSigner),
+		stop:               make(chan struct{}),
 	}, nil
 }
 
 // Watch will reconcile the sentries with the kube api at a reasonable interval.
 // It must be called only once.
 func (w *SentryWatcher) Watch(ctx context.Context) {
+	for _, sentry := range w.persistentSentries {
+		if err := sentry.Start(); err != nil {
+			w.log.Error("Failed to start persistent sentry", "error", err)
+		}
+	}
+	if !w.operator {
+		return
+	}
 	defer close(w.done)
 	const interval = 30 * time.Second
 	timer := time.NewTimer(interval)
@@ -110,6 +133,9 @@ func (w *SentryWatcher) Stop() error {
 	close(w.stop)
 	<-w.done
 	var err error
+	for _, sentry := range w.persistentSentries {
+		err = errors.Join(err, sentry.Stop())
+	}
 	for _, sentry := range w.sentries {
 		err = errors.Join(err, sentry.Stop())
 	}
@@ -194,7 +220,7 @@ func (w *SentryWatcher) reconcileSentries(
 
 	for _, newSentry := range newSentries {
 		dialer := net.Dialer{Timeout: 2 * time.Second}
-		s := signer.NewReconnRemoteSigner(newSentry, w.log, w.lb, dialer)
+		s := signer.NewReconnRemoteSigner(newSentry, w.log, w.hc, dialer)
 
 		if err := s.Start(); err != nil {
 			return fmt.Errorf("failed to start new remote signer(s): %w", err)

diff --git a/go.mod b/go.mod
@@ -4,56 +4,126 @@ go 1.20
 
 require (
 	github.com/cometbft/cometbft v0.37.2
-	github.com/cosmos/gogoproto v1.4.1
+	github.com/cosmos/gogoproto v1.4.10
 	github.com/spf13/cobra v1.6.1
+	github.com/strangelove-ventures/horcrux v0.1.5-0.20231108213903-6aab5001533f
 	github.com/stretchr/testify v1.8.2
-	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
+	golang.org/x/sync v0.1.0
+	google.golang.org/grpc v1.55.0
 	k8s.io/apimachinery v0.28.1
 	k8s.io/client-go v0.28.1
 )
 
 require (
-	github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
+	cosmossdk.io/errors v1.0.0-beta.7 // indirect
+	cosmossdk.io/math v1.0.1 // indirect
+	filippo.io/edwards25519 v1.0.0 // indirect
+	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
+	github.com/Jille/raft-grpc-leader-rpc v1.1.0 // indirect
+	github.com/Jille/raft-grpc-transport v1.4.0 // indirect
+	github.com/Jille/raftadmin v1.2.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cometbft/cometbft-db v0.7.0 // indirect
+	github.com/confio/ics23/go v0.9.0 // indirect
+	github.com/cosmos/btcutil v1.0.5 // indirect
+	github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
+	github.com/cosmos/cosmos-sdk v0.47.3 // indirect
+	github.com/cosmos/go-bip39 v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
+	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/ethereum/go-ethereum v1.12.0 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/google/btree v1.1.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gtank/merlin v0.1.1 // indirect
+	github.com/gtank/ristretto255 v0.1.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/raft v1.5.0 // indirect
+	github.com/hashicorp/raft-boltdb/v2 v2.2.2 // indirect
+	github.com/hdevalence/ed25519consensus v0.1.0 // indirect
+	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.7 // indirect
+	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.14.0 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
+	github.com/tendermint/go-amino v0.16.0 // indirect
+	gitlab.com/unit410/edwards25519 v0.0.0-20220725154547-61980033348e // indirect
+	gitlab.com/unit410/threshold-ed25519 v0.0.0-20220725172740-6ee731f539ac // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
 	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/net v0.13.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/term v0.10.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.28.1 // indirect