Feat add addrbook

api/v1/cosmosfullnode_types.go

@@ -445,6 +445,27 @@ type ChainSpec struct {
 	// +optional
 	LogFormat *string `json:"logFormat"`
 
+	// URL to address book file to download from the internet.
+	// The operator detects and properly handles the following file extensions:
+	// .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip
+	// Use AddrbookScript if the chain has an unconventional file format or address book location.
+	// +optional
+	AddrbookURL *string `json:"addrbookURL"`
+
+	// Specify shell (sh) script commands to properly download and save the address book file.
+	// Prefer AddrbookURL if the file is in a conventional format.
+	// The available shell commands are from docker image ghcr.io/strangelove-ventures/infra-toolkit, including wget and curl.
+	// Save the file to env var $ADDRBOOK_FILE.
+	// E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE
+	// Takes precedence over AddrbookURL.
+	// Hint: Use "set -eux" in your script.
+	// Available env vars:
+	// $HOME: The home directory.
+	// $ADDRBOOK_FILE: The location of the final address book file.
+	// $CONFIG_DIR: The location of the config dir that houses the address book file. Used for extracting from archives. The archive must have a single file called "addrbook.json".
+	// +optional
+	AddrbookScript *string `json:"addrbookScript"`
+
 	// URL to genesis file to download from the internet.
 	// Although this field is optional, you will almost always want to set it.
 	// If not set, uses the genesis file created from the init subcommand. (This behavior may be desirable for new chains or testing.)

config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml

@@ -45,6 +45,27 @@ spec:
               chain:
                 description: Blockchain-specific configuration.
                 properties:
+                  addrbookScript:
+                    description: 'Specify shell (sh) script commands to properly download
+                      and save the address book file. Prefer AddrbookURL if the file
+                      is in a conventional format. The available shell commands are
+                      from docker image ghcr.io/strangelove-ventures/infra-toolkit,
+                      including wget and curl. Save the file to env var $ADDRBOOK_FILE.
+                      E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE Takes
+                      precedence over AddrbookURL. Hint: Use "set -eux" in your script.
+                      Available env vars: $HOME: The home directory. $ADDRBOOK_FILE:
+                      The location of the final address book file. $CONFIG_DIR: The
+                      location of the config dir that houses the address book file.
+                      Used for extracting from archives. The archive must have a single
+                      file called "addrbook.json".'
+                    type: string
+                  addrbookURL:
+                    description: 'URL to address book file to download from the internet.
+                      The operator detects and properly handles the following file
+                      extensions: .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip
+                      Use AddrbookScript if the chain has an unconventional file format
+                      or address book location.'
+                    type: string
                   app:
                     description: App configuration applied to app.toml.
                     properties:

internal/fullnode/addrbook.go

@@ -0,0 +1,40 @@
+package fullnode
+
+import (
+	_ "embed"
+	"fmt"
+
+	cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
+)
+
+var (
+	//go:embed script/download-addrbook.sh
+	scriptDownloadAddrbook string
+)
+
+const addrbookScriptWrapper = `ls $CONFIG_DIR/addrbook.json 1> /dev/null 2>&1
+ADDRBOOK_EXISTS=$?
+if [ $ADDRBOOK_EXISTS -eq 0 ]; then
+	echo "Address book already exists"
+	exit 0
+fi
+ls -l $CONFIG_DIR/addrbook.json 
+%s
+ls -l $CONFIG_DIR/addrbook.json 
+
+echo "Address book $ADDRBOOK_FILE downloaded"
+`
+
+// DownloadGenesisCommand returns a proper address book command for use in an init container.
+func DownloadAddrbookCommand(cfg cosmosv1.ChainSpec) (string, []string) {
+	args := []string{"-c"}
+	switch {
+	case cfg.AddrbookScript != nil:
+		args = append(args, fmt.Sprintf(addrbookScriptWrapper, *cfg.AddrbookScript))
+	case cfg.AddrbookURL != nil:
+		args = append(args, fmt.Sprintf(addrbookScriptWrapper, scriptDownloadAddrbook), "-s", *cfg.AddrbookURL)
+	default:
+		args = append(args, "echo Using default address book")
+	}
+	return "sh", args
+}

internal/fullnode/addrbook_test.go

@@ -0,0 +1,72 @@
+package fullnode
+
+import (
+	"testing"
+
+	cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadAddrbookCommand(t *testing.T) {
+	t.Parallel()
+
+	requireValidScript := func(t *testing.T, script string) {
+		t.Helper()
+		require.NotEmpty(t, script)
+		require.Contains(t, script, `if [ $ADDRBOOK_EXISTS -eq 0 ]`)
+	}
+
+	t.Run("default", func(t *testing.T) {
+		var cfg cosmosv1.ChainSpec
+
+		cmd, args := DownloadAddrbookCommand(cfg)
+		require.Equal(t, "sh", cmd)
+
+		require.Len(t, args, 2)
+
+		require.Equal(t, "-c", args[0])
+
+		got := args[1]
+		require.NotContains(t, got, "ADDRBOOK_EXISTS")
+		require.Contains(t, got, "Using default address book")
+	})
+
+	t.Run("download", func(t *testing.T) {
+		cfg := cosmosv1.ChainSpec{
+			AddrbookURL: ptr("https://example.com/addrbook.json"),
+		}
+		cmd, args := DownloadAddrbookCommand(cfg)
+		require.Equal(t, "sh", cmd)
+
+		require.Len(t, args, 4)
+
+		require.Equal(t, "-c", args[0])
+		got := args[1]
+		requireValidScript(t, got)
+		require.Contains(t, got, `ADDRBOOK_URL`)
+		require.Contains(t, got, "download_json")
+
+		require.Equal(t, "-s", args[2])
+		require.Equal(t, "https://example.com/addrbook.json", args[3])
+	})
+
+	t.Run("custom", func(t *testing.T) {
+		cfg := cosmosv1.ChainSpec{
+			// Keeping this to assert that custom script takes precedence.
+			AddrbookURL:    ptr("https://example.com/addrbook.json"),
+			AddrbookScript: ptr("echo hi"),
+		}
+		cmd, args := DownloadAddrbookCommand(cfg)
+		require.Equal(t, "sh", cmd)
+
+		require.Len(t, args, 2)
+
+		require.Equal(t, "-c", args[0])
+
+		got := args[1]
+		requireValidScript(t, got)
+
+		require.NotContains(t, got, "ADDRBOOK_URL")
+		require.Contains(t, got, "echo hi")
+	})
+}

internal/fullnode/pod_builder.go

@@ -278,6 +278,7 @@ func envVars(crd *cosmosv1.CosmosFullNode) []corev1.EnvVar {
 		{Name: "HOME", Value: workDir},
 		{Name: "CHAIN_HOME", Value: home},
 		{Name: "GENESIS_FILE", Value: path.Join(home, "config", "genesis.json")},
+		{Name: "ADDRBOOK_FILE", Value: path.Join(home, "config", "addrbook.json")},
 		{Name: "CONFIG_DIR", Value: path.Join(home, "config")},
 		{Name: "DATA_DIR", Value: path.Join(home, "data")},
 	}
@@ -287,6 +288,7 @@ func initContainers(crd *cosmosv1.CosmosFullNode, moniker string) []corev1.Conta
 	tpl := crd.Spec.PodTemplate
 	binary := crd.Spec.ChainSpec.Binary
 	genesisCmd, genesisArgs := DownloadGenesisCommand(crd.Spec.ChainSpec)
+	addrbookCmd, addrbookArgs := DownloadAddrbookCommand(crd.Spec.ChainSpec)
 	env := envVars(crd)
 
 	initCmd := fmt.Sprintf("%s init %s --chain-id %s", binary, moniker, crd.Spec.ChainSpec.ChainID)
@@ -332,7 +334,15 @@ echo "Initializing into tmp dir for downstream processing..."
 			ImagePullPolicy: tpl.ImagePullPolicy,
 			WorkingDir:      workDir,
 		},
-
+		{
+			Name:            "addrbook-init",
+			Image:           infraToolImage,
+			Command:         []string{addrbookCmd},
+			Args:            addrbookArgs,
+			Env:             env,
+			ImagePullPolicy: tpl.ImagePullPolicy,
+			WorkingDir:      workDir,
+		},
 		{
 			Name:    "config-merge",
 			Image:   infraToolImage,

internal/fullnode/pod_builder_test.go

@@ -222,10 +222,12 @@ func TestPodBuilder(t *testing.T) {
 		require.Equal(t, startContainer.Env[1].Value, "/home/operator/cosmos")
 		require.Equal(t, startContainer.Env[2].Name, "GENESIS_FILE")
 		require.Equal(t, startContainer.Env[2].Value, "/home/operator/cosmos/config/genesis.json")
-		require.Equal(t, startContainer.Env[3].Name, "CONFIG_DIR")
-		require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config")
-		require.Equal(t, startContainer.Env[4].Name, "DATA_DIR")
-		require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/data")
+		require.Equal(t, startContainer.Env[3].Name, "ADDRBOOK_FILE")
+		require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config/addrbook.json")
+		require.Equal(t, startContainer.Env[4].Name, "CONFIG_DIR")
+		require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/config")
+		require.Equal(t, startContainer.Env[5].Name, "DATA_DIR")
+		require.Equal(t, startContainer.Env[5].Value, "/home/operator/cosmos/data")
 		require.Equal(t, envVars(&crd), startContainer.Env)
 
 		healthContainer := pod.Spec.Containers[1]
@@ -242,14 +244,15 @@ func TestPodBuilder(t *testing.T) {
 		}
 		require.Equal(t, healthPort, healthContainer.Ports[0])
 
-		require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 5)
+		require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 6)
 
 		wantInitImages := []string{
 			"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
 			"main-image:v1.2.3",
 			"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
 			"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
 			"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
+			"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
 		}
 		require.Equal(t, wantInitImages, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string {
 			return c.Image
@@ -267,7 +270,11 @@ func TestPodBuilder(t *testing.T) {
 		require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$CHAIN_HOME"`)
 		require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$HOME/.tmp"`)
 
-		mergeConfig := pod.Spec.InitContainers[3]
+		mergeConfig1 := pod.Spec.InitContainers[3]
+		// The order of config-merge arguments is important. Rightmost takes precedence.
+		require.Contains(t, mergeConfig1.Args[1], `echo Using default address book`)
+
+		mergeConfig := pod.Spec.InitContainers[4]
 		// The order of config-merge arguments is important. Rightmost takes precedence.
 		require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/config.toml" "$OVERLAY_DIR/config-overlay.toml" > "$CONFIG_DIR/config.toml"`)
 		require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/app.toml" "$OVERLAY_DIR/app-overlay.toml" > "$CONFIG_DIR/app.toml`)
@@ -293,10 +300,12 @@ func TestPodBuilder(t *testing.T) {
 		require.Equal(t, container.Env[1].Value, "/home/operator/.osmosisd")
 		require.Equal(t, container.Env[2].Name, "GENESIS_FILE")
 		require.Equal(t, container.Env[2].Value, "/home/operator/.osmosisd/config/genesis.json")
-		require.Equal(t, container.Env[3].Name, "CONFIG_DIR")
-		require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config")
-		require.Equal(t, container.Env[4].Name, "DATA_DIR")
-		require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/data")
+		require.Equal(t, container.Env[3].Name, "ADDRBOOK_FILE")
+		require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config/addrbook.json")
+		require.Equal(t, container.Env[4].Name, "CONFIG_DIR")
+		require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/config")
+		require.Equal(t, container.Env[5].Name, "DATA_DIR")
+		require.Equal(t, container.Env[5].Value, "/home/operator/.osmosisd/data")
 
 		require.NotEmpty(t, pod.Spec.InitContainers)
 
@@ -554,7 +563,7 @@ gaiad start --home /home/operator/cosmos`
 		require.Equal(t, "/foo", extraVol[0].MountPath)
 
 		initConts := lo.SliceToMap(pod.Spec.InitContainers, func(c corev1.Container) (string, corev1.Container) { return c.Name, c })
-		require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "config-merge"}, lo.Keys(initConts))
+		require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "addrbook-init", "config-merge"}, lo.Keys(initConts))
 		require.Equal(t, "foo:latest", initConts["chain-init"].Image)
 	})
 

internal/fullnode/script/download-addrbook.sh

@@ -0,0 +1,53 @@
+set -eu
+
+# $ADDRBOOK_FILE and $CONFIG_DIR already set via pod env vars.
+
+ADDRBOOK_URL="$1"
+
+echo "Downloading address book file $ADDRBOOK_URL to $ADDRBOOK_FILE..."
+
+download_json() {
+  echo "Downloading plain json..."
+  wget -c -O "$ADDRBOOK_FILE" "$ADDRBOOK_URL"
+}
+
+download_jsongz() {
+  echo "Downloading json.gz..."
+  wget -c -O - "$ADDRBOOK_URL" | gunzip -c >"$ADDRBOOK_FILE"
+}
+
+download_tar() {
+  echo "Downloading and extracting tar..."
+  wget -c -O - "$ADDRBOOK_URL" | tar -x -C "$CONFIG_DIR"
+}
+
+download_targz() {
+  echo "Downloading and extracting compressed tar..."
+  wget -c -O - "$ADDRBOOK_URL" | tar -xz -C "$CONFIG_DIR"
+}
+
+download_zip() {
+  echo "Downloading and extracting zip..."
+  wget -c -O tmp_genesis.zip "$ADDRBOOK_URL"
+  unzip tmp_genesis.zip
+  rm tmp_genesis.zip
+  mv genesis.json "$ADDRBOOK_FILE"
+}
+
+rm -f "$ADDRBOOK_FILE"
+
+case "$ADDRBOOK_URL" in
+*.json.gz) download_jsongz ;;
+*.json) download_json ;;
+*.tar.gz) download_targz ;;
+*.tar.gzip) download_targz ;;
+*.tar) download_tar ;;
+*.zip) download_zip ;;
+*)
+  echo "Unable to handle file extension for $ADDRBOOK_URL"
+  exit 1
+  ;;
+esac
+
+echo "Saved address book file to $ADDRBOOK_FILE."
+echo "Download address book file complete."