diff --git a/.gitignore b/.gitignore index be6ebce..2be5b7e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ go.work.sum .env .vscode + +configs/secrets* +!configs/secrets.example.yml \ No newline at end of file diff --git a/README.md b/README.md index 3215e4a..4e881d3 100644 --- a/README.md +++ b/README.md @@ -1 +1,193 @@ -# indexer \ No newline at end of file +# indexer + +## Configuration + +You can configure the application in 3 ways. +The order of priority is +1. Command line arguments +2. Environment variables +3. Configuration files + +### Configuration using command line arguments +You can configure the application using command line arguments. +For example to configure the `rpc.url` configuration, you can use the `--rpc-url` command line argument. +Only select arguments are implemented. More on this below. + +### Configuration using environment variables +You can also configure the application using environment variables. You can configure any configuration in the `config.yml` file using environment variables by replacing the `.` in the configuration with `_` and making the variable name uppercase. +For example to configure the `rpc.url` configuration to `https://my-rpc.com`, you can set the `RPC_URL` environment variable to `https://my-rpc.com`. + +### Configuration using configuration files +The default configuration file is `configs/config.yml`. You can use the `--config` flag to specify a different configuration file. +If you want to add secrets to the configuration file, you can copy `configs/secrets.example.yml` to `configs/secrets.yml` and add the secrets. They won't be commited to the repository or the built image. + +### Supported configurations: + +#### RPC URL +URL to use as the RPC client. + +cmd: `--rpc-url` +env: `RPC_URL` +yaml: +```yaml +rpc: + url: https://rpc.com +``` + +#### Log Level +Log level for the logger. Default is `warn`. + +cmd: `--log-level` +env: `LOG_LEVEL` +yaml: +```yaml +log: + level: debug +``` + +#### Pretty log +Whether to print logs in a pretty format. Affects performance. Default is `false`. + +env: `PRETTY_LOG` +yaml: +```yaml +log: + pretty: true +``` + +#### Poller +Whether to enable the poller. Default is `true`. + +cmd: `--poller` +env: `POLLER_ENABLED` +yaml: +```yaml +poller: + enabled: true +``` + +#### Poller Interval +Poller trigger interval in millisecons. Default is `1000`. + +env: `POLLER_INTERVAL` +yaml: +```yaml +poller: + interval: 3000 +``` + +#### Poller Batch Size +How many blocks to poll each interval. Default is `10`. + +env: `POLLER_BATCH_SIZE` +yaml: +```yaml +poller: + batchSize: 3 +``` + +#### Poller From Block +From which block to start polling. Default is `0`. + +env: `POLLER_FROM_BLOCK` +yaml: +```yaml +poller: + fromBlock: 20000000 +``` + +#### Poller Until Block +Until which block to poll. If not set, it will poll until the latest block. + +env: `POLLER_UNTIL_BLOCK` +yaml: +```yaml +poller: + untilBlock: 20000010 +``` + +#### Commiter +Whether to enable the commiter. Default is `true`. + +cmd: `--commiter` +env: `COMMITER_ENABLED` +yaml: +```yaml +commiter: + enabled: true +``` + +#### Commiter Interval +Commiter trigger interval in millisecons. Default is `250`. + +env: `COMMITER_INTERVAL` +yaml: +```yaml +commiter: + interval: 3000 +``` + +#### Commiter Batch Size +How many blocks to commit each interval. Default is `10`. + +env: `COMMITER_BATCH_SIZE` +yaml: +```yaml +commiter: + batchSize: 3 +``` + +#### Failure Recoverer +Whether to enable the failure recoverer. Default is `true`. + +cmd: `--failure-recoverer` +env: `FAILURE_RECOVERER_ENABLED` +yaml: +```yaml +failureRecoverer: + enabled: true +``` + +#### Failure Recoverer Interval +Failure recoverer trigger interval in millisecons. Default is `1000`. + +env: `FAILURE_RECOVERER_INTERVAL` +yaml: +```yaml +failureRecoverer: + interval: 3000 +``` + +#### Failure Recoverer Batch Size +How many blocks to recover each interval. Default is `10`. + +env: `FAILURE_RECOVERER_BATCH_SIZE` +yaml: +```yaml +failureRecoverer: + batchSize: 3 +``` + +#### Storage +This application has 3 kinds of strorage. Main, Staging and Orchestrator. +Each of them takes similar configuration depending on the driver you want to use. + +There are no defaults, so this needs to be configured. + +```yaml +storage: + main: + driver: "clickhouse" + clickhouse: + host: "localhost" + port: 3000 + user: "admin" + password: "admin" + database: "base" + staging: + driver: "memory" + memory: + maxItems: 1000000 + orchestrator: + driver: "memory" +``` \ No newline at end of file diff --git a/cmd/api.go b/cmd/api.go new file mode 100644 index 0000000..dceb1ed --- /dev/null +++ b/cmd/api.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "net/http" + + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/go-chi/chi/v5" + "github.com/thirdweb-dev/indexer/internal/handlers" +) + +var ( + apiCmd = &cobra.Command{ + Use: "api", + Short: "TBD", + Long: "TBD", + Run: func(cmd *cobra.Command, args []string) { + var r *chi.Mux = chi.NewRouter() + handlers.Handler(r) + + log.Info().Msg("Starting Server on port 3000") + err := http.ListenAndServe("localhost:3000", r) + if err != nil { + log.Error().Err(err).Msg("Error starting server") + } + }, + } +) diff --git a/cmd/api/__debug_bin3998373950 b/cmd/api/__debug_bin3998373950 deleted file mode 100755 index fa41ba4..0000000 Binary files a/cmd/api/__debug_bin3998373950 and /dev/null differ diff --git a/cmd/api/main.go b/cmd/api/main.go deleted file mode 100644 index 5ee506c..0000000 --- a/cmd/api/main.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "net/http" - - "github.com/go-chi/chi/v5" - "github.com/rs/zerolog/log" - "github.com/thirdweb-dev/indexer/internal/env" - "github.com/thirdweb-dev/indexer/internal/handlers" - customLogger "github.com/thirdweb-dev/indexer/internal/log" -) - -func main() { - env.Load() - customLogger.InitLogger() - - var r *chi.Mux = chi.NewRouter() - handlers.Handler(r) - - log.Info().Msg("Starting Server on port 3000") - err := http.ListenAndServe("localhost:3000", r) - if err != nil { - log.Error().Err(err).Msg("Error starting server") - } -} diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go deleted file mode 100644 index 2436826..0000000 --- a/cmd/indexer/main.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "github.com/rs/zerolog/log" - "github.com/thirdweb-dev/indexer/internal/common" - "github.com/thirdweb-dev/indexer/internal/env" - customLogger "github.com/thirdweb-dev/indexer/internal/log" - "github.com/thirdweb-dev/indexer/internal/orchestrator" -) - -func main() { - env.Load() - customLogger.InitLogger() - - log.Info().Msg("Starting indexer") - rpc, err := common.InitializeRPC() - if err != nil { - log.Fatal().Err(err).Msg("Failed to initialize RPC") - } - - orchestrator, err := orchestrator.NewOrchestrator(*rpc) - if err != nil { - log.Fatal().Err(err).Msg("Failed to create orchestrator") - } - - orchestrator.Start() -} diff --git a/cmd/orchestrator.go b/cmd/orchestrator.go new file mode 100644 index 0000000..d06f8ac --- /dev/null +++ b/cmd/orchestrator.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/thirdweb-dev/indexer/internal/common" + "github.com/thirdweb-dev/indexer/internal/orchestrator" +) + +var ( + orchestratorCmd = &cobra.Command{ + Use: "orchestrator", + Short: "TBD", + Long: "TBD", + Run: func(cmd *cobra.Command, args []string) { + log.Info().Msg("Starting indexer") + rpc, err := common.InitializeRPC() + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize RPC") + } + + orchestrator, err := orchestrator.NewOrchestrator(*rpc) + if err != nil { + log.Fatal().Err(err).Msg("Failed to create orchestrator") + } + + orchestrator.Start() + }, + } +) diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..d9c11c8 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "os" + + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" + configs "github.com/thirdweb-dev/indexer/configs" + customLogger "github.com/thirdweb-dev/indexer/internal/log" +) + +var ( + // Used for flags. + cfgFile string + + rootCmd = &cobra.Command{ + Use: "indexer", + Short: "TBD", + Long: "TBD", + Run: func(cmd *cobra.Command, args []string) { + log.Info().Msg("TODO: Starting indexer & api both") + }, + } +) + +func Execute() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is ./configs/config.yml)") + rootCmd.PersistentFlags().String("rpc-url", "", "RPC Url to use for the indexer") + rootCmd.PersistentFlags().String("log-level", "", "Log level to use for the application") + rootCmd.PersistentFlags().Bool("poller", true, "Toggle poller") + rootCmd.PersistentFlags().Bool("commiter", true, "Toggle commiter") + rootCmd.PersistentFlags().Bool("failure-recoverer", true, "Toggle failure recoverer") + viper.BindPFlag("rpc.url", rootCmd.PersistentFlags().Lookup("rpc-url")) + viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) + viper.BindPFlag("poller.enabled", rootCmd.PersistentFlags().Lookup("poller")) + viper.BindPFlag("commiter.enabled", rootCmd.PersistentFlags().Lookup("commiter")) + viper.BindPFlag("failure-recoverer.enabled", rootCmd.PersistentFlags().Lookup("failure-recoverer")) + + rootCmd.AddCommand(orchestratorCmd) + rootCmd.AddCommand(apiCmd) +} + +func initConfig() { + configs.LoadConfig(cfgFile) + customLogger.InitLogger() +} diff --git a/configs/config.go b/configs/config.go new file mode 100644 index 0000000..39fe49e --- /dev/null +++ b/configs/config.go @@ -0,0 +1,113 @@ +package config + +import ( + "fmt" + "strings" + + "github.com/spf13/viper" +) + +type LogConfig struct { + Level string `mapstructure:"level"` + Pretty bool `mapstructure:"pretty"` +} + +type PollerConfig struct { + Enabled bool `mapstructure:"enabled"` + Interval int `mapstructure:"interval"` + BatchSize int `mapstructure:"batchSize"` + FromBlock int `mapstructure:"fromBlock"` + UntilBlock int `mapstructure:"untilBlock"` +} + +type CommitterConfig struct { + Enabled bool `mapstructure:"enabled"` + Interval int `mapstructure:"interval"` + BatchSize int `mapstructure:"batchSize"` +} + +type FailureRecovererConfig struct { + Enabled bool `mapstructure:"enabled"` + Interval int `mapstructure:"interval"` + BatchSize int `mapstructure:"batchSize"` +} + +type StorageConfig struct { + Staging StorageConnectionConfig `mapstructure:"staging"` + Main StorageConnectionConfig `mapstructure:"main"` + Orchestrator StorageConnectionConfig `mapstructure:"orchestrator"` +} +type StorageType string + +const ( + StorageTypeMain StorageType = "main" + StorageTypeStaging StorageType = "staging" + StorageTypeOrchestrator StorageType = "orchestrator" +) + +type StorageConnectionConfig struct { + Clickhouse *ClickhouseConfig `mapstructure:"clickhouse"` + Memory *MemoryConfig `mapstructure:"memory"` +} + +type ClickhouseConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` +} + +type MemoryConfig struct { + MaxItems int `mapstructure:"maxItems"` +} + +type RPCConfig struct { + URL string `mapstructure:"url"` +} + +type Config struct { + RPC RPCConfig `mapstructure:"rpc"` + Log LogConfig `mapstructure:"log"` + Poller PollerConfig `mapstructure:"poller"` + Committer CommitterConfig `mapstructure:"committer"` + FailureRecoverer FailureRecovererConfig `mapstructure:"failureRecoverer"` + Storage StorageConfig `mapstructure:"storage"` +} + +var Cfg Config + +func LoadConfig(cfgFile string) error { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + if err := viper.ReadInConfig(); err != nil { + return fmt.Errorf("error reading config file, %s", err) + } + } else { + viper.SetConfigName("config") + viper.AddConfigPath("./configs") + + if err := viper.ReadInConfig(); err != nil { + return fmt.Errorf("error reading config file, %s", err) + } + + viper.SetConfigName("secrets") + err := viper.MergeInConfig() + if err != nil { + return fmt.Errorf("error loading secrets file: %v", err) + } + } + + // sets e.g. RPC_URL to rpc.url + replacer := strings.NewReplacer(".", "_") + viper.SetEnvKeyReplacer(replacer) + + viper.AutomaticEnv() + + err := viper.Unmarshal(&Cfg) + if err != nil { + return fmt.Errorf("error unmarshalling config: %v", err) + } + + return nil +} diff --git a/configs/config.yml b/configs/config.yml new file mode 100644 index 0000000..ebb8b52 --- /dev/null +++ b/configs/config.yml @@ -0,0 +1,33 @@ +log: + level: debug + pretty: true + +poller: + enabled: true + interval: 3000 + batchSize: 3 + fromBlock: 20000000 + untilBlock: 20000010 + +commiter: + enabled: true + interval: 10000 + batchSize: 100 + +failureRecoverer: + enabled: true + interval: 3000 + batchSize: 100 + +storage: + main: + clickhouse: + port: 9440 + database: "base" + staging: + clickhouse: + port: 9440 + database: "staging" + orchestrator: + memory: + maxItems: 10000 \ No newline at end of file diff --git a/configs/secrets.example.yml b/configs/secrets.example.yml new file mode 100644 index 0000000..80c12b5 --- /dev/null +++ b/configs/secrets.example.yml @@ -0,0 +1,14 @@ +rpc: + url: https://1.rpc.thirdweb.com + +storage: + main: + clickhouse: + host: localhost + username: admin + password: admin + staging: + clickhouse: + host: localhost + username: admin + password: admin \ No newline at end of file diff --git a/go.mod b/go.mod index 4e89243..ecf005e 100644 --- a/go.mod +++ b/go.mod @@ -23,34 +23,54 @@ require ( github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.3.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.7 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/paulmach/orb v0.11.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shopspring/decimal v1.4.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/otel v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.17.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 0e000a2..1715b99 100644 --- a/go.sum +++ b/go.sum @@ -39,13 +39,16 @@ github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5U github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= @@ -58,8 +61,8 @@ github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= @@ -103,6 +106,8 @@ github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpx github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -111,6 +116,8 @@ github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -131,6 +138,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -141,8 +150,8 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= @@ -154,12 +163,15 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= @@ -179,18 +191,42 @@ github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -219,6 +255,8 @@ go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -280,6 +318,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/internal/common/rpc.go b/internal/common/rpc.go index f1f0a52..b3e349e 100644 --- a/internal/common/rpc.go +++ b/internal/common/rpc.go @@ -4,12 +4,12 @@ import ( "context" "fmt" "math/big" - "os" "strings" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/rs/zerolog/log" + config "github.com/thirdweb-dev/indexer/configs" ) type RPC struct { @@ -22,11 +22,12 @@ type RPC struct { } func InitializeRPC() (*RPC, error) { - rpcURL := os.Getenv("RPC_URL") - if rpcURL == "" { + rpcUrl := config.Cfg.RPC.URL + if rpcUrl == "" { return nil, fmt.Errorf("RPC_URL environment variable is not set") } - rpcClient, dialErr := rpc.Dial(rpcURL) + log.Debug().Msgf("Initializing RPC with URL: %s", rpcUrl) + rpcClient, dialErr := rpc.Dial(rpcUrl) if dialErr != nil { return nil, dialErr } @@ -36,8 +37,8 @@ func InitializeRPC() (*RPC, error) { rpc := &RPC{ RPCClient: rpcClient, EthClient: ethClient, - URL: rpcURL, - IsWebsocket: strings.HasPrefix(rpcURL, "ws://") || strings.HasPrefix(rpcURL, "wss://"), + URL: rpcUrl, + IsWebsocket: strings.HasPrefix(rpcUrl, "ws://") || strings.HasPrefix(rpcUrl, "wss://"), } checkErr := rpc.checkSupportedMethods() if checkErr != nil { diff --git a/internal/env/load.go b/internal/env/load.go deleted file mode 100644 index e41ea38..0000000 --- a/internal/env/load.go +++ /dev/null @@ -1,13 +0,0 @@ -package env - -import ( - "github.com/joho/godotenv" - "github.com/rs/zerolog/log" -) - -func Load() { - err := godotenv.Load() - if err != nil { - log.Error().Err(err).Msg("error loading .env file") - } -} diff --git a/internal/log/logger.go b/internal/log/logger.go index 73781cf..b605f02 100644 --- a/internal/log/logger.go +++ b/internal/log/logger.go @@ -6,6 +6,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/rs/zerolog/pkgerrors" + config "github.com/thirdweb-dev/indexer/configs" ) func InitLogger() { @@ -17,15 +18,14 @@ func NewLogger(name string) zerolog.Logger { zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack level := zerolog.WarnLevel - if lvl, err := zerolog.ParseLevel(os.Getenv("LOG_LEVEL")); err == nil && lvl != zerolog.NoLevel { + if lvl, err := zerolog.ParseLevel(config.Cfg.Log.Level); err == nil && lvl != zerolog.NoLevel { level = lvl } zerolog.SetGlobalLevel(level) - prettify := os.Getenv("LOG_PRETTIFY") == "true" logger := zerolog.New(os.Stderr).With().Timestamp().Str("component", name).Logger() logger = logger.With().Caller().Logger() - if prettify { + if config.Cfg.Log.Pretty { logger = logger.Output(zerolog.ConsoleWriter{Out: os.Stderr}) } return logger diff --git a/internal/orchestrator/commiter.go b/internal/orchestrator/commiter.go index 06704b9..fd3e571 100644 --- a/internal/orchestrator/commiter.go +++ b/internal/orchestrator/commiter.go @@ -3,13 +3,12 @@ package orchestrator import ( "fmt" "math/big" - "os" "sort" - "strconv" "sync" "time" "github.com/rs/zerolog/log" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" "github.com/thirdweb-dev/indexer/internal/storage" ) @@ -25,24 +24,20 @@ type Commiter struct { } func NewCommiter(storage storage.IStorage) *Commiter { - triggerInterval, err := strconv.Atoi(os.Getenv("COMMITER_TRIGGER_INTERVAL")) - if err != nil || triggerInterval == 0 { + triggerInterval := config.Cfg.Committer.Interval + if triggerInterval == 0 { triggerInterval = DEFAULT_COMMITER_TRIGGER_INTERVAL } - blocksPerCommit, err := strconv.Atoi(os.Getenv("BLOCKS_PER_COMMIT")) - if err != nil || blocksPerCommit == 0 { + blocksPerCommit := config.Cfg.Committer.BatchSize + if blocksPerCommit == 0 { blocksPerCommit = DEFAULT_BLOCKS_PER_COMMIT } - pollFromBlock, err := strconv.ParseUint(os.Getenv("POLL_FROM_BLOCK"), 10, 64) - if err != nil { - pollFromBlock = 0 - } return &Commiter{ triggerIntervalMs: triggerInterval, blocksPerCommit: blocksPerCommit, storage: storage, - pollFromBlock: big.NewInt(int64(pollFromBlock)), + pollFromBlock: big.NewInt(int64(config.Cfg.Poller.FromBlock)), } } diff --git a/internal/orchestrator/failure_recoverer.go b/internal/orchestrator/failure_recoverer.go index 40fba9d..5fbe1cf 100644 --- a/internal/orchestrator/failure_recoverer.go +++ b/internal/orchestrator/failure_recoverer.go @@ -2,11 +2,10 @@ package orchestrator import ( "math/big" - "os" - "strconv" "time" "github.com/rs/zerolog/log" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" "github.com/thirdweb-dev/indexer/internal/storage" "github.com/thirdweb-dev/indexer/internal/worker" @@ -23,12 +22,12 @@ type FailureRecoverer struct { } func NewFailureRecoverer(rpc common.RPC, storage storage.IStorage) *FailureRecoverer { - failuresPerPoll, err := strconv.Atoi(os.Getenv("FAILURES_PER_POLL")) - if err != nil || failuresPerPoll == 0 { + failuresPerPoll := config.Cfg.FailureRecoverer.BatchSize + if failuresPerPoll == 0 { failuresPerPoll = DEFAULT_FAILURES_PER_POLL } - triggerInterval, err := strconv.Atoi(os.Getenv("FAILURE_TRIGGER_INTERVAL")) - if err != nil || triggerInterval == 0 { + triggerInterval := config.Cfg.FailureRecoverer.Interval + if triggerInterval == 0 { triggerInterval = DEFAULT_FAILURE_TRIGGER_INTERVAL } return &FailureRecoverer{ diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index f13f15a..ed5a1e7 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -1,9 +1,9 @@ package orchestrator import ( - "os" "sync" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" "github.com/thirdweb-dev/indexer/internal/storage" ) @@ -17,17 +17,7 @@ type Orchestrator struct { } func NewOrchestrator(rpc common.RPC) (*Orchestrator, error) { - storage, err := storage.NewStorageConnector(&storage.StorageConfig{ - Orchestrator: storage.ConnectorConfig{Driver: "memory"}, - Main: storage.ConnectorConfig{Driver: "clickhouse", Clickhouse: &storage.ClickhouseConnectorConfig{ - Database: "base", - Table: "base", - }}, - Staging: storage.ConnectorConfig{Driver: "clickhouse", Clickhouse: &storage.ClickhouseConnectorConfig{ - Database: "staging", - Table: "staging", - }}, - }) + storage, err := storage.NewStorageConnector(&config.Cfg.Storage) if err != nil { return nil, err } @@ -35,9 +25,9 @@ func NewOrchestrator(rpc common.RPC) (*Orchestrator, error) { return &Orchestrator{ rpc: rpc, storage: storage, - pollerEnabled: os.Getenv("DISABLE_POLLER") != "true", - failureRecovererEnabled: os.Getenv("DISABLE_FAILURE_RECOVERY") != "true", - committerEnabled: os.Getenv("DISABLE_COMMITTER") != "true", + pollerEnabled: config.Cfg.Poller.Enabled, + failureRecovererEnabled: config.Cfg.FailureRecoverer.Enabled, + committerEnabled: config.Cfg.Committer.Enabled, }, nil } diff --git a/internal/orchestrator/poller.go b/internal/orchestrator/poller.go index b7f524c..aab8664 100644 --- a/internal/orchestrator/poller.go +++ b/internal/orchestrator/poller.go @@ -4,11 +4,10 @@ import ( "context" "fmt" "math/big" - "os" - "strconv" "time" "github.com/rs/zerolog/log" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" "github.com/thirdweb-dev/indexer/internal/storage" "github.com/thirdweb-dev/indexer/internal/worker" @@ -33,29 +32,21 @@ type BlockNumberWithError struct { } func NewPoller(rpc common.RPC, storage storage.IStorage) *Poller { - blocksPerPoll, err := strconv.Atoi(os.Getenv("BLOCKS_PER_POLL")) - if err != nil || blocksPerPoll == 0 { + blocksPerPoll := config.Cfg.Poller.BatchSize + if blocksPerPoll == 0 { blocksPerPoll = DEFAULT_BLOCKS_PER_POLL } - triggerInterval, err := strconv.Atoi(os.Getenv("TRIGGER_INTERVAL")) - if err != nil || triggerInterval == 0 { + triggerInterval := config.Cfg.Poller.Interval + if triggerInterval == 0 { triggerInterval = DEFAULT_TRIGGER_INTERVAL } - pollUntilBlock, err := strconv.ParseUint(os.Getenv("POLL_UNTIL_BLOCK"), 10, 64) - if err != nil { - pollUntilBlock = 0 - } - pollFromBlock, err := strconv.ParseUint(os.Getenv("POLL_FROM_BLOCK"), 10, 64) - if err != nil { - pollFromBlock = 0 - } return &Poller{ rpc: rpc, triggerIntervalMs: int64(triggerInterval), blocksPerPoll: int64(blocksPerPoll), storage: storage, - pollUntilBlock: big.NewInt(int64(pollUntilBlock)), - pollFromBlock: big.NewInt(int64(pollFromBlock)), + pollUntilBlock: big.NewInt(int64(config.Cfg.Poller.UntilBlock)), + pollFromBlock: big.NewInt(int64(config.Cfg.Poller.FromBlock)), } } diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index b2a6e8e..f340c1b 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -6,28 +6,21 @@ import ( "encoding/json" "fmt" "math/big" - "os" - "strconv" "time" "github.com/ClickHouse/clickhouse-go/v2" zLog "github.com/rs/zerolog/log" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" ) type ClickHouseConnector struct { conn clickhouse.Conn - cfg *ClickhouseConnectorConfig + cfg *config.ClickhouseConfig } -type ClickhouseConnectorConfig struct { - ExpiresAt time.Duration - Database string - Table string -} - -func NewClickHouseConnector(cfg *ClickhouseConnectorConfig) (*ClickHouseConnector, error) { - conn, err := connectDB() +func NewClickHouseConnector(cfg *config.ClickhouseConfig) (*ClickHouseConnector, error) { + conn, err := connectDB(cfg) // Question: Should we add the table setup here? if err != nil { return nil, err @@ -76,19 +69,19 @@ func (c *ClickHouseConnector) Delete(index, partitionKey, rangeKey string) error return nil } -func connectDB() (clickhouse.Conn, error) { - port, err := strconv.Atoi(os.Getenv("CLICKHOUSE_PORT")) - if err != nil { - return nil, fmt.Errorf("invalid CLICKHOUSE_PORT: %w", err) +func connectDB(cfg *config.ClickhouseConfig) (clickhouse.Conn, error) { + port := cfg.Port + if port == 0 { + return nil, fmt.Errorf("invalid CLICKHOUSE_PORT: %d", port) } conn, err := clickhouse.Open(&clickhouse.Options{ - Addr: []string{fmt.Sprintf("%s:%d", os.Getenv("CLICKHOUSE_HOST"), port)}, + Addr: []string{fmt.Sprintf("%s:%d", cfg.Host, port)}, Protocol: clickhouse.Native, TLS: &tls.Config{}, // enable secure TLS Auth: clickhouse.Auth{ - Username: os.Getenv("CLICKHOUSE_USERNAME"), - Password: os.Getenv("CLICKHOUSE_PASSWORD"), + Username: cfg.Username, + Password: cfg.Password, }, }) if err != nil { diff --git a/internal/storage/connector.go b/internal/storage/connector.go index 827a4f1..2505357 100644 --- a/internal/storage/connector.go +++ b/internal/storage/connector.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" ) @@ -13,18 +14,6 @@ type QueryFilter struct { Offset uint64 } -type StorageConfig struct { - Main ConnectorConfig - Staging ConnectorConfig - Orchestrator ConnectorConfig -} - -type ConnectorConfig struct { - Driver string - Memory *MemoryConnectorConfig - Clickhouse *ClickhouseConnectorConfig -} - type IStorage struct { OrchestratorStorage IOrchestratorStorage MainStorage IMainStorage @@ -59,21 +48,21 @@ type IMainStorage interface { GetMaxBlockNumber() (maxBlockNumber *big.Int, err error) } -func NewStorageConnector(cfg *StorageConfig) (IStorage, error) { +func NewStorageConnector(cfg *config.StorageConfig) (IStorage, error) { var storage IStorage var err error - storage.OrchestratorStorage, err = newConnector[IOrchestratorStorage](cfg.Orchestrator) + storage.OrchestratorStorage, err = newConnector[IOrchestratorStorage](&cfg.Orchestrator) if err != nil { return IStorage{}, fmt.Errorf("failed to create orchestrator storage: %w", err) } - storage.MainStorage, err = newConnector[IMainStorage](cfg.Main) + storage.MainStorage, err = newConnector[IMainStorage](&cfg.Main) if err != nil { return IStorage{}, fmt.Errorf("failed to create main storage: %w", err) } - storage.StagingStorage, err = newConnector[IStagingStorage](cfg.Staging) + storage.StagingStorage, err = newConnector[IStagingStorage](&cfg.Staging) if err != nil { return IStorage{}, fmt.Errorf("failed to create staging storage: %w", err) } @@ -81,16 +70,15 @@ func NewStorageConnector(cfg *StorageConfig) (IStorage, error) { return storage, nil } -func newConnector[T any](cfg ConnectorConfig) (T, error) { +func newConnector[T any](cfg *config.StorageConnectionConfig) (T, error) { var conn interface{} var err error - switch cfg.Driver { - case "memory": - conn, err = NewMemoryConnector(cfg.Memory) - case "clickhouse": + if cfg.Clickhouse != nil { conn, err = NewClickHouseConnector(cfg.Clickhouse) - default: - return *new(T), fmt.Errorf("invalid connector driver: %s", cfg.Driver) + } else if cfg.Memory != nil { + conn, err = NewMemoryConnector(cfg.Memory) + } else { + return *new(T), fmt.Errorf("no storage driver configured") } if err != nil { diff --git a/internal/storage/memory.go b/internal/storage/memory.go index 452ad58..38f3f41 100644 --- a/internal/storage/memory.go +++ b/internal/storage/memory.go @@ -8,20 +8,17 @@ import ( "strings" lru "github.com/hashicorp/golang-lru/v2" + config "github.com/thirdweb-dev/indexer/configs" "github.com/thirdweb-dev/indexer/internal/common" ) -type MemoryConnectorConfig struct { - MaxItems int -} - type MemoryConnector struct { cache *lru.Cache[string, string] } -func NewMemoryConnector(cfg *MemoryConnectorConfig) (*MemoryConnector, error) { +func NewMemoryConnector(cfg *config.MemoryConfig) (*MemoryConnector, error) { maxItems := 1000 - if cfg != nil && cfg.MaxItems > 0 { + if cfg.MaxItems > 0 { maxItems = cfg.MaxItems } diff --git a/main.go b/main.go new file mode 100644 index 0000000..e859b5a --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/thirdweb-dev/indexer/cmd" + +func main() { + cmd.Execute() +}