[BLOCK-1845] add configured firehose ratelimit

ultraio · Nov 2, 2023 · bebd655 · bebd655
1 parent 5c74f12
commit bebd655
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 3 deletions.
diff --git a/cmd/dfuseeos/cli/firehose.go b/cmd/dfuseeos/cli/firehose.go
@@ -40,6 +40,8 @@ func init() {
 			cmd.Flags().String("common-network-name", "ultra", "Common network name such as eos, wax, ...")
 			cmd.Flags().String("firehose-grpc-listen-addr", FirehoseGRPCServingAddr+"*", "Address on which the firehose will listen")
 			cmd.Flags().StringSlice("firehose-blocks-store-urls", nil, "If non-empty, overrides common-blocks-store-url with a list of blocks stores")
+			/*ultra-duncan --- BLOCK-1845 --- set default rate limit for firehose*/
+			cmd.Flags().Int("firehose-ratelimit", 1000, "Rate limit for how many doc can be processed per second")
 			return nil
 		},
 
@@ -103,12 +105,19 @@ func init() {
 				return nil, fmt.Errorf("missing common-network-name")
 			}
 
+			/*ultra-duncan --- BLOCK-1845 --- set default rate limit for firehose*/
+			ratelimit := viper.GetInt("firehose-ratelimit")
+			if ratelimit <= 0 {
+				return nil, fmt.Errorf("firehose-ratelimit must be positive")
+			}
+
 			return firehoseApp.New(appLogger, &firehoseApp.Config{
 				BlockStoreURLs:          firehoseBlocksStoreURLs,
 				BlockStreamAddr:         blockstreamAddr,
 				GRPCListenAddr:          viper.GetString("firehose-grpc-listen-addr"),
 				GRPCShutdownGracePeriod: grcpShutdownGracePeriod,
 				Network:                 networkName,
+				RateLimit:               ratelimit,
 			}, &firehoseApp.Modules{
 				Authenticator:             authenticator,
 				BlockTrimmer:              blockstreamv2.BlockTrimmerFunc(trimBlock),

diff --git a/go.mod b/go.mod
@@ -232,7 +232,7 @@ replace (
 	github.com/streamingfast/dauth => github.com/EOS-Nation/dauth v0.0.0-20221005084741-e79ebf7b78e8
 	github.com/streamingfast/dgrpc => github.com/EOS-Nation/dgrpc v0.0.0-20221108170744-d0c2d8abe98f
 	github.com/streamingfast/dstore => github.com/EOS-Nation/dstore v0.0.0-20220908095022-20cd13d5dc4c
-	github.com/streamingfast/firehose => github.com/ultraio/firehose v0.0.0-20230928055645-1f99a6d31876
+	github.com/streamingfast/firehose => github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303
 )
 
 // replace github.com/eoscanada/eos-go => github.com/EOS-Nation/eos-go v0.10.3-0.20230328111622-b58e29c1532e

diff --git a/go.sum b/go.sum
@@ -966,8 +966,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
 github.com/ultraio/eos-go v0.9.1-0.20230721092210-7ab11fa47e4f h1:qT+BLkcestDui1StBCD27M/3Sueb2hCWj2DdSQPE4/Y=
 github.com/ultraio/eos-go v0.9.1-0.20230721092210-7ab11fa47e4f/go.mod h1:BCTBH+XkEi+gO38MHRDm0LLcUhaIlekey+SCBaIAJTk=
-github.com/ultraio/firehose v0.0.0-20230928055645-1f99a6d31876 h1:cGL9FeeGdKHeHNwxaQS23GruCJ+nujLS+YFxDRqzzGg=
-github.com/ultraio/firehose v0.0.0-20230928055645-1f99a6d31876/go.mod h1:MO8mgZUEOLgCfc8jIfRIT/XD3Tw6Xxg++qx7XXQUMQA=
+github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303 h1:NYOAN/a5DC2esgIpfk9+bmqmkVhHHnZ61vur9jqhsB0=
+github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303/go.mod h1:MO8mgZUEOLgCfc8jIfRIT/XD3Tw6Xxg++qx7XXQUMQA=
 github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
 github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=