From bebd655b562ca133bc5a6063c5a9b2c48001fca2 Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Thu, 2 Nov 2023 10:30:19 +0700 Subject: [PATCH 1/2] [BLOCK-1845] add configured firehose ratelimit --- cmd/dfuseeos/cli/firehose.go | 9 +++++++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/dfuseeos/cli/firehose.go b/cmd/dfuseeos/cli/firehose.go index a428ffbf..7d0b99d2 100644 --- a/cmd/dfuseeos/cli/firehose.go +++ b/cmd/dfuseeos/cli/firehose.go @@ -40,6 +40,8 @@ func init() { cmd.Flags().String("common-network-name", "ultra", "Common network name such as eos, wax, ...") cmd.Flags().String("firehose-grpc-listen-addr", FirehoseGRPCServingAddr+"*", "Address on which the firehose will listen") cmd.Flags().StringSlice("firehose-blocks-store-urls", nil, "If non-empty, overrides common-blocks-store-url with a list of blocks stores") + /*ultra-duncan --- BLOCK-1845 --- set default rate limit for firehose*/ + cmd.Flags().Int("firehose-ratelimit", 1000, "Rate limit for how many doc can be processed per second") return nil }, @@ -103,12 +105,19 @@ func init() { return nil, fmt.Errorf("missing common-network-name") } + /*ultra-duncan --- BLOCK-1845 --- set default rate limit for firehose*/ + ratelimit := viper.GetInt("firehose-ratelimit") + if ratelimit <= 0 { + return nil, fmt.Errorf("firehose-ratelimit must be positive") + } + return firehoseApp.New(appLogger, &firehoseApp.Config{ BlockStoreURLs: firehoseBlocksStoreURLs, BlockStreamAddr: blockstreamAddr, GRPCListenAddr: viper.GetString("firehose-grpc-listen-addr"), GRPCShutdownGracePeriod: grcpShutdownGracePeriod, Network: networkName, + RateLimit: ratelimit, }, &firehoseApp.Modules{ Authenticator: authenticator, BlockTrimmer: blockstreamv2.BlockTrimmerFunc(trimBlock), diff --git a/go.mod b/go.mod index c8e17a44..9266fb7f 100644 --- a/go.mod +++ b/go.mod @@ -232,7 +232,7 @@ replace ( github.com/streamingfast/dauth => github.com/EOS-Nation/dauth v0.0.0-20221005084741-e79ebf7b78e8 github.com/streamingfast/dgrpc => github.com/EOS-Nation/dgrpc v0.0.0-20221108170744-d0c2d8abe98f github.com/streamingfast/dstore => github.com/EOS-Nation/dstore v0.0.0-20220908095022-20cd13d5dc4c - github.com/streamingfast/firehose => github.com/ultraio/firehose v0.0.0-20230928055645-1f99a6d31876 + github.com/streamingfast/firehose => github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303 ) // replace github.com/eoscanada/eos-go => github.com/EOS-Nation/eos-go v0.10.3-0.20230328111622-b58e29c1532e diff --git a/go.sum b/go.sum index 4296b328..678455cd 100644 --- a/go.sum +++ b/go.sum @@ -966,8 +966,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/ultraio/eos-go v0.9.1-0.20230721092210-7ab11fa47e4f h1:qT+BLkcestDui1StBCD27M/3Sueb2hCWj2DdSQPE4/Y= github.com/ultraio/eos-go v0.9.1-0.20230721092210-7ab11fa47e4f/go.mod h1:BCTBH+XkEi+gO38MHRDm0LLcUhaIlekey+SCBaIAJTk= -github.com/ultraio/firehose v0.0.0-20230928055645-1f99a6d31876 h1:cGL9FeeGdKHeHNwxaQS23GruCJ+nujLS+YFxDRqzzGg= -github.com/ultraio/firehose v0.0.0-20230928055645-1f99a6d31876/go.mod h1:MO8mgZUEOLgCfc8jIfRIT/XD3Tw6Xxg++qx7XXQUMQA= +github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303 h1:NYOAN/a5DC2esgIpfk9+bmqmkVhHHnZ61vur9jqhsB0= +github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303/go.mod h1:MO8mgZUEOLgCfc8jIfRIT/XD3Tw6Xxg++qx7XXQUMQA= github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4= github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= From f5ea5e2cb67de2ab7351f757b8c09a900447815b Mon Sep 17 00:00:00 2001 From: Duncan Dam Date: Thu, 2 Nov 2023 10:45:52 +0700 Subject: [PATCH 2/2] update commit hash for firehose --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9266fb7f..f6ae9d1a 100644 --- a/go.mod +++ b/go.mod @@ -232,7 +232,7 @@ replace ( github.com/streamingfast/dauth => github.com/EOS-Nation/dauth v0.0.0-20221005084741-e79ebf7b78e8 github.com/streamingfast/dgrpc => github.com/EOS-Nation/dgrpc v0.0.0-20221108170744-d0c2d8abe98f github.com/streamingfast/dstore => github.com/EOS-Nation/dstore v0.0.0-20220908095022-20cd13d5dc4c - github.com/streamingfast/firehose => github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303 + github.com/streamingfast/firehose => github.com/ultraio/firehose v0.0.0-20231102034459-8ba0e223932c ) // replace github.com/eoscanada/eos-go => github.com/EOS-Nation/eos-go v0.10.3-0.20230328111622-b58e29c1532e diff --git a/go.sum b/go.sum index 678455cd..7f67c5ba 100644 --- a/go.sum +++ b/go.sum @@ -966,8 +966,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/ultraio/eos-go v0.9.1-0.20230721092210-7ab11fa47e4f h1:qT+BLkcestDui1StBCD27M/3Sueb2hCWj2DdSQPE4/Y= github.com/ultraio/eos-go v0.9.1-0.20230721092210-7ab11fa47e4f/go.mod h1:BCTBH+XkEi+gO38MHRDm0LLcUhaIlekey+SCBaIAJTk= -github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303 h1:NYOAN/a5DC2esgIpfk9+bmqmkVhHHnZ61vur9jqhsB0= -github.com/ultraio/firehose v0.0.0-20231102032842-cffc973eb303/go.mod h1:MO8mgZUEOLgCfc8jIfRIT/XD3Tw6Xxg++qx7XXQUMQA= +github.com/ultraio/firehose v0.0.0-20231102034459-8ba0e223932c h1:BJF6IO8MyEb7U+NvAYevXbaJuzsRbAnKHfmekEUWHXI= +github.com/ultraio/firehose v0.0.0-20231102034459-8ba0e223932c/go.mod h1:MO8mgZUEOLgCfc8jIfRIT/XD3Tw6Xxg++qx7XXQUMQA= github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4= github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=