feat: scripts to generate boot snapshots for testnet/preprod (#118)

.dockerignore

@@ -11,3 +11,4 @@ sidecar.db*
 go-sidecar
 !/sqlite-extensions
 /internal/tests/testdata
+/snapshots

.github/workflows/check-merge.yml

@@ -0,0 +1,36 @@
+name: check-pr-merge
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches:
+      - master
+      - "release/*"
+jobs:
+  check-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Check if branch is up to date
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          baseRef="origin/${BASE_REF}"
+          echo "Targeting '$baseRef'"
+          
+          git fetch origin
+          
+          res=$(git --no-pager log HEAD..$baseRef --oneline)
+          if [[ -z "$res" ]]; then
+            echo "Branch is up to date with master"
+          else
+            echo "Branch is not up to date with master"
+            echo "Please rebase your branch"
+            echo "$res"
+            exit 1
+          fi

.gitignore

@@ -29,5 +29,6 @@ node_modules
 .release_version
 chart_releases
 /snapshots/**/*.sql
+/snapshots/**/*.csv
 *.sql
 *.dump

cmd/database.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/Layr-Labs/sidecar/internal/config"
+	"github.com/Layr-Labs/sidecar/internal/logger"
+	"github.com/Layr-Labs/sidecar/internal/metrics"
+	"github.com/Layr-Labs/sidecar/internal/version"
+	"github.com/Layr-Labs/sidecar/pkg/clients/ethereum"
+	"github.com/Layr-Labs/sidecar/pkg/contractCaller/sequentialContractCaller"
+	"github.com/Layr-Labs/sidecar/pkg/contractManager"
+	"github.com/Layr-Labs/sidecar/pkg/contractStore/postgresContractStore"
+	"github.com/Layr-Labs/sidecar/pkg/eigenState"
+	"github.com/Layr-Labs/sidecar/pkg/eigenState/stateManager"
+	"github.com/Layr-Labs/sidecar/pkg/fetcher"
+	"github.com/Layr-Labs/sidecar/pkg/indexer"
+	"github.com/Layr-Labs/sidecar/pkg/pipeline"
+	"github.com/Layr-Labs/sidecar/pkg/postgres"
+	"github.com/Layr-Labs/sidecar/pkg/postgres/migrations"
+	"github.com/Layr-Labs/sidecar/pkg/rewards"
+	pgStorage "github.com/Layr-Labs/sidecar/pkg/storage/postgres"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+	"log"
+)
+
+var runDatabaseCmd = &cobra.Command{
+	Use:   "database",
+	Short: "Database utility",
+	Run: func(cmd *cobra.Command, args []string) {
+		initDatabaseCmd(cmd)
+		cfg := config.NewConfig()
+
+		l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
+
+		l.Sugar().Infow("sidecar run",
+			zap.String("version", version.GetVersion()),
+			zap.String("commit", version.GetCommit()),
+		)
+
+		sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
+		if err != nil {
+			l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
+		}
+
+		client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
+
+		pgConfig := postgres.PostgresConfigFromDbConfig(&cfg.DatabaseConfig)
+
+		pg, err := postgres.NewPostgres(pgConfig)
+		if err != nil {
+			l.Fatal("Failed to setup postgres connection", zap.Error(err))
+		}
+
+		grm, err := postgres.NewGormFromPostgresConnection(pg.Db)
+		if err != nil {
+			l.Fatal("Failed to create gorm instance", zap.Error(err))
+		}
+
+		migrator := migrations.NewMigrator(pg.Db, grm, l)
+		if err = migrator.MigrateAll(); err != nil {
+			l.Fatal("Failed to migrate", zap.Error(err))
+		}
+
+		contractStore := postgresContractStore.NewPostgresContractStore(grm, l, cfg)
+		if err := contractStore.InitializeCoreContracts(); err != nil {
+			log.Fatalf("Failed to initialize core contracts: %v", err)
+		}
+
+		cm := contractManager.NewContractManager(contractStore, client, sdc, l)
+
+		mds := pgStorage.NewPostgresBlockStore(grm, l, cfg)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		sm := stateManager.NewEigenStateManager(l, grm)
+
+		if err := eigenState.LoadEigenStateModels(sm, grm, l, cfg); err != nil {
+			l.Sugar().Fatalw("Failed to load eigen state models", zap.Error(err))
+		}
+
+		fetchr := fetcher.NewFetcher(client, cfg, l)
+
+		cc := sequentialContractCaller.NewSequentialContractCaller(client, cfg, l)
+
+		idxr := indexer.NewIndexer(mds, contractStore, cm, client, fetchr, cc, grm, l, cfg)
+
+		rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, l)
+		if err != nil {
+			l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
+		}
+
+		_ = pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, l)
+
+		l.Sugar().Infow("Done")
+	},
+}
+
+func initDatabaseCmd(cmd *cobra.Command) {
+	cmd.Flags().VisitAll(func(f *pflag.Flag) {
+		if err := viper.BindPFlag(config.KebabToSnakeCase(f.Name), f); err != nil {
+			fmt.Printf("Failed to bind flag '%s' - %+v\n", f.Name, err)
+		}
+		if err := viper.BindEnv(f.Name); err != nil {
+			fmt.Printf("Failed to bind env '%s' - %+v\n", f.Name, err)
+		}
+
+	})
+}

cmd/root.go

@@ -50,6 +50,7 @@ func init() {
 	rootCmd.AddCommand(runCmd)
 	rootCmd.AddCommand(runOperatorRestakedStrategiesCmd)
 	rootCmd.AddCommand(runVersionCmd)
+	rootCmd.AddCommand(runDatabaseCmd)
 }
 
 func initConfig(cmd *cobra.Command) {

scripts/seedSnapshot.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+DATABASE=$1
+NETWORK=$2
+
+if [[ -z $DATABASE ]]; then
+    echo "Usage: $0 <database> <network>"
+    exit 1
+fi
+
+if [[ -z $NETWORK ]]; then
+    echo "Usage: $0 <database> <network>"
+    exit 1
+fi
+
+networks=("mainnet" "testnet" "preprod")
+if [[ ! " ${networks[@]} " =~ " ${NETWORK} " ]]; then
+    echo "Invalid network"
+    exit 1
+fi
+
+dropdb $DATABASE || true
+createdb $DATABASE
+
+go run main.go database \
+    --ethereum.rpc-url="http://34.229.43.36:8545" \
+    --chain=$NETWORK \
+    --statsd.url="localhost:8125" \
+    --database.host="localhost" \
+    --database.port="5432" \
+    --database.user=$PG_USER \
+    --database.password=$PG_PASSWORD \
+    --database.db_name=$DATABASE
+
+sourcesPath="snapshots/$NETWORK/sources"
+
+for i in $(ls $sourcesPath | grep '.sql'); do
+    full_file="${sourcesPath}/$i"
+    echo $full_file
+    psql --dbname $DATABASE < $full_file
+done
+
+goldFile="$(pwd)/snapshots/${NETWORK}/sources/sidecar-${NETWORK}-holesky-gold.csv"
+echo "Loading gold table from: $goldFile"
+
+psql -c "\copy gold_table from '${goldFile}' CSV HEADER;" --dbname $DATABASE
+
+psql -c "insert into state_roots (eth_block_number, eth_block_hash, state_root) select number as eth_block_number, hash as eth_block_hash, 'first state root' as state_root from blocks where number = (select max(number) from blocks);" --dbname $DATABASE
+
+

scripts/takeSnapshot.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+DBNAME=$1
+NETWORK=$2
+SCHEMA=$3
+DESTINATION=$4
+
+_usage="Usage: $0 <dbname> <network> <schema> <destination>"
+
+if [[ -z $DBNAME ]]; then
+    echo $_usage
+    exit 1
+fi
+
+if [[ -z $NETWORK ]]; then
+    echo $_usage
+    exit 1
+fi
+
+networks=("mainnet" "testnet" "preprod")
+if [[ ! " ${networks[@]} " =~ " ${NETWORK} " ]]; then
+    echo "Invalid network"
+    exit 1
+fi
+
+if [[ -z $SCHEMA ]]; then
+    echo $_usage
+    exit 1
+fi
+
+if [[ -z $DESTINATION ]]; then
+    echo $_usage
+    exit 1
+fi
+
+echo "Snapshotting database $DBNAME"
+
+pg_dump --schema $SCHEMA -Fc $DBNAME > $DESTINATION
+
+echo "Done!"