diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index e344af72..a71f5bb5 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/setup-go@v3 with: - go-version: 1.21 + go-version: 1.22 - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6737780e..48e47103 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/setup-go@v3 with: - go-version: 1.21 + go-version: 1.22 - uses: actions/checkout@v3 - name: Cache Go modules uses: actions/cache@v2 diff --git a/.golangci.yml b/.golangci.yml index 2a0f6187..abb8d2c3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -58,6 +58,6 @@ linters: - wsl run: - go: '1.21' + go: '1.22' timeout: 5m issues-exit-code: 1 diff --git a/Dockerfile b/Dockerfile index e51b041a..679875bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 -FROM golang:1.21-alpine3.20 as builder +FROM golang:1.22.6-alpine3.20 as builder RUN apk add --no-cache make libc6-compat build-base diff --git a/README.md b/README.md index 7430b9fe..3e7c4def 100644 --- a/README.md +++ b/README.md @@ -107,11 +107,21 @@ go build -v ./... go test -v ./... ``` +Running all tests that match regexp `TestHealthCheckManager`: +```sh +go test -v -run TestHealthCheckManager ./... +``` + To measure test code coverage, install the following tool and run the above `go test` command with the `-cover` flag: ```sh go get golang.org/x/tools/cmd/cover ``` +You can generate HTML coverage report and open it in your browser by running: +```sh +go test -v -coverprofile=cover.out ./... && go tool cover -html=cover.out +``` + #### Linting This project relies on [golangci-lint](https://github.com/golangci/golangci-lint) for linting. You can set up an [integration with your code editor](https://golangci-lint.run/usage/integrations/) to run lint checks locally. diff --git a/go.mod b/go.mod index 32d94949..898352f4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,9 @@ toolchain go1.22.3 require ( github.com/ethereum/go-ethereum v1.12.0 + github.com/failsafe-go/failsafe-go v0.6.8 github.com/go-redis/cache/v9 v9.0.0 + github.com/go-redis/redismock/v9 v9.0.3 github.com/google/go-cmp v0.6.0 github.com/prometheus/client_golang v1.14.0 github.com/redis/go-redis/v9 v9.0.5 @@ -20,68 +22,43 @@ require ( require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chigopher/pathlib v0.19.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-ole/go-ole v1.2.1 // indirect - github.com/go-redis/redismock/v9 v9.0.3 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect - github.com/huandu/xstrings v1.5.0 // indirect - github.com/iancoleman/strcase v0.3.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jinzhu/copier v0.4.0 // indirect github.com/klauspost/compress v1.17.2 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.39.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/rs/zerolog v1.33.0 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.19.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect - github.com/vektra/mockery/v2 v2.43.2 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.19.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.23.0 // indirect - golang.org/x/tools/cmd/cover v0.1.0-deprecated // indirect - google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 2d7d3041..a80713e7 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,41 @@ github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chigopher/pathlib v0.19.1 h1:RoLlUJc0CqBGwq239cilyhxPNLXTK+HXoASGyGznx5A= -github.com/chigopher/pathlib v0.19.1/go.mod h1:tzC1dZLW8o33UQpWkNkhvPwL5n4yyFRFm/jL1YGWFvY= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -42,14 +49,18 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/failsafe-go/failsafe-go v0.6.8 h1:ERrJMknjXdtDVrx1s05uE5MCDhGTiF7rQ98z6bdVUOw= +github.com/failsafe-go/failsafe-go v0.6.8/go.mod h1:LAo0yJE2PXn1z4T22bkmUxPryrTHUvMhvnwik9x2uq8= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -60,10 +71,12 @@ github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= +github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -73,79 +86,66 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= -github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= -github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -169,11 +169,8 @@ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2 github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -189,66 +186,43 @@ github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wO github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= -github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= -github.com/vektra/mockery/v2 v2.43.2 h1:OdivAsQL/uoQ55UnTt25tliRI8kaj5j6caHk9xaAUD0= -github.com/vektra/mockery/v2 v2.43.2/go.mod h1:XNTE9RIu3deGAGQRVjP1VZxGpQNm0YedZx4oDs3prr8= +github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= @@ -256,16 +230,14 @@ github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= @@ -274,12 +246,9 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -287,8 +256,6 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -304,15 +271,14 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -336,15 +302,11 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -353,8 +315,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -362,11 +322,10 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -374,16 +333,14 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= -golang.org/x/tools/cmd/cover v0.1.0-deprecated h1:Rwy+mWYz6loAF+LnG1jHG/JWMHRMMC2/1XX3Ejkx9lA= -golang.org/x/tools/cmd/cover v0.1.0-deprecated/go.mod h1:hMDiIvlpN1NoVgmjLjUJE9tMHyxHjFX7RuQ+rW12mSA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -393,19 +350,17 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -413,7 +368,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/checks/checks_test.go b/internal/checks/checks_test.go index 4929b95f..dd501d2c 100644 --- a/internal/checks/checks_test.go +++ b/internal/checks/checks_test.go @@ -3,6 +3,7 @@ package checks import ( "errors" "testing" + "time" "github.com/satsuma-data/node-gateway/internal/config" "github.com/stretchr/testify/assert" @@ -14,6 +15,42 @@ var defaultUpstreamConfig = &config.UpstreamConfig{ WSURL: "wss://alchemy", } +var defaultRoutingConfig = &config.RoutingConfig{ + PassiveLatencyChecking: true, + DetectionWindow: config.NewDuration(10 * time.Minute), + BanWindow: config.NewDuration(50 * time.Minute), + Errors: &config.ErrorsConfig{ + Rate: 0.25, + HTTPCodes: []string{ + "5xx", + "420", + }, + JSONRPCCodes: []string{ + "32xxx", + }, + ErrorStrings: []string{ + "internal server error", + }, + }, + Latency: &config.LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "eth_call": 10000 * time.Millisecond, + "eth_getLogs": 2000 * time.Millisecond, + }, + Threshold: 1000 * time.Millisecond, + Methods: []config.MethodConfig{ + { + Name: "eth_getLogs", + Threshold: 2000 * time.Millisecond, + }, + { + Name: "eth_call", + Threshold: 10000 * time.Millisecond, + }, + }, + }, +} + type methodNotSupportedError struct{} func (e methodNotSupportedError) Error() string { return "Method Not Supported." } diff --git a/internal/checks/latency.go b/internal/checks/latency.go new file mode 100644 index 00000000..0dafe172 --- /dev/null +++ b/internal/checks/latency.go @@ -0,0 +1,484 @@ +package checks + +import ( + "context" + "math" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/failsafe-go/failsafe-go/circuitbreaker" + + "github.com/satsuma-data/node-gateway/internal/client" + conf "github.com/satsuma-data/node-gateway/internal/config" + "github.com/satsuma-data/node-gateway/internal/metrics" + "github.com/satsuma-data/node-gateway/internal/types" + "go.uber.org/zap" +) + +const ( + ResponseCodeWildcard = 'x' + PercentPerFrac = 100 + MinNumRequestsForRate = 1 // The minimum number of requests required to compute the error rate. +) + +type ErrorCircuitBreaker interface { + RecordRequest(isError bool) + IsOpen() bool +} + +type LatencyCircuitBreaker interface { + RecordLatency(latency time.Duration) + IsOpen() bool + GetThreshold() time.Duration +} + +type ErrorStats struct { + circuitBreaker circuitbreaker.CircuitBreaker[any] +} + +func (e *ErrorStats) RecordRequest(isError bool) { + if isError { + e.circuitBreaker.RecordFailure() + } else { + e.circuitBreaker.RecordSuccess() + } +} + +func (e *ErrorStats) IsOpen() bool { + // TODO(polsar): Dedup with LatencyStats.IsOpen. + return e.circuitBreaker.IsOpen() || e.circuitBreaker.IsHalfOpen() +} + +func NewErrorStats(routingConfig *conf.RoutingConfig) ErrorCircuitBreaker { + return &ErrorStats{ + circuitBreaker: NewCircuitBreaker( + getErrorsRate(routingConfig), + getDetectionWindow(routingConfig), + getBanWindow(routingConfig), + ), + } +} + +type LatencyStats struct { + circuitBreaker circuitbreaker.CircuitBreaker[any] + threshold time.Duration +} + +func (l *LatencyStats) RecordLatency(latency time.Duration) { + if latency >= l.threshold { + l.circuitBreaker.RecordFailure() + } else { + l.circuitBreaker.RecordSuccess() + } +} + +func (l *LatencyStats) IsOpen() bool { + // TODO(polsar): Dedup with ErrorStats.IsOpen. + return l.circuitBreaker.IsOpen() || l.circuitBreaker.IsHalfOpen() +} + +func (l *LatencyStats) GetThreshold() time.Duration { + return l.threshold +} + +func NewLatencyStats(routingConfig *conf.RoutingConfig, method string) LatencyCircuitBreaker { + return &LatencyStats{ + threshold: getLatencyThreshold(routingConfig, method), + circuitBreaker: NewCircuitBreaker( + conf.DefaultLatencyTooHighRate, + getDetectionWindow(routingConfig), + getBanWindow(routingConfig), + ), + } +} + +// NewCircuitBreaker abstracts away the rather complex circuitbreaker.Builder API. +// https://pkg.go.dev/github.com/failsafe-go/failsafe-go/circuitbreaker +// https://failsafe-go.dev/circuit-breaker/ +func NewCircuitBreaker( + errorRate float64, + detectionWindow time.Duration, + banWindow time.Duration, +) circuitbreaker.CircuitBreaker[any] { + // TODO(polsar): Check that `0.0 < errorRate <= 1.0` holds. + return circuitbreaker.Builder[any](). + HandleResult(false). // The false return value of the wrapped call will be interpreted as a failure. + WithFailureRateThreshold( + uint(math.Floor(errorRate*PercentPerFrac)), // Minimum percentage of failed requests to open the breaker. + MinNumRequestsForRate, + detectionWindow, + ). + WithDelay(banWindow). + Build() +} + +type LatencyCheck struct { + client client.EthClient + Err error + clientGetter client.EthClientGetter + metricsContainer *metrics.Container + logger *zap.Logger + upstreamConfig *conf.UpstreamConfig + routingConfig *conf.RoutingConfig + errorCircuitBreaker ErrorCircuitBreaker + methodLatencyBreaker map[string]LatencyCircuitBreaker // RPC method -> LatencyCircuitBreaker + lock sync.RWMutex + ShouldRun bool +} + +func NewLatencyChecker( + upstreamConfig *conf.UpstreamConfig, + routingConfig *conf.RoutingConfig, + clientGetter client.EthClientGetter, + metricsContainer *metrics.Container, + logger *zap.Logger, +) types.LatencyChecker { + c := &LatencyCheck{ + upstreamConfig: upstreamConfig, + routingConfig: routingConfig, + clientGetter: clientGetter, + metricsContainer: metricsContainer, + logger: logger, + errorCircuitBreaker: NewErrorStats(routingConfig), + methodLatencyBreaker: make(map[string]LatencyCircuitBreaker), + ShouldRun: routingConfig.Errors != nil || routingConfig.Latency != nil, + } + + if err := c.InitializePassiveCheck(); err != nil { + logger.Error("Error initializing LatencyCheck.", zap.Any("upstreamID", c.upstreamConfig), zap.Error(err)) + } + + return c +} + +func (c *LatencyCheck) InitializePassiveCheck() error { + // TODO(polsar): Set `c.ShouldRun` if active health checking is enabled. + c.logger.Debug("Initializing LatencyCheck.", zap.Any("config", c.upstreamConfig)) + + httpClient, err := c.clientGetter(c.upstreamConfig.HTTPURL, &c.upstreamConfig.BasicAuthConfig, &c.upstreamConfig.RequestHeadersConfig) + if err != nil { + c.Err = err + return c.Err + } + + c.client = httpClient + + c.runPassiveCheck() + + // TODO(polsar): This check is in both PeerCheck and SyncingCheck implementations, so refactor this. + if isMethodNotSupportedErr(c.Err) { + c.logger.Debug("LatencyCheck is not supported by upstream, not running check.", zap.String("upstreamID", c.upstreamConfig.ID)) + + c.ShouldRun = false + } + + return nil +} + +func (c *LatencyCheck) RunPassiveCheck() { + if c.client == nil { + if err := c.InitializePassiveCheck(); err != nil { + c.logger.Error("Error initializing LatencyCheck.", zap.Any("upstreamID", c.upstreamConfig.ID), zap.Error(err)) + c.metricsContainer.LatencyCheckErrors.WithLabelValues(c.upstreamConfig.ID, c.upstreamConfig.HTTPURL, metrics.HTTPInit).Inc() + } + } + + if c.ShouldRun { + c.runPassiveCheck() + } +} + +func (c *LatencyCheck) runPassiveCheck() { + if c.client == nil || !c.routingConfig.PassiveLatencyChecking { + return + } + + latencyConfig := c.routingConfig.Latency + if latencyConfig == nil { + return + } + + var wg sync.WaitGroup + defer wg.Wait() + + // Iterate over all (method, latencyThreshold) pairs and launch the check for each in parallel. + // Note that `latencyConfig.MethodLatencyThresholds` is never modified after its initialization + // in `config` package, so we don't need a lock to protect concurrent read access. + for method, latencyThreshold := range latencyConfig.MethodLatencyThresholds { + wg.Add(1) + + // Passing the loop variables as arguments is required to prevent the following lint error: + // loopclosure: loop variable method captured by func literal (govet) + go func(method string, latencyThreshold time.Duration) { + defer wg.Done() + + runCheck := func() { + c.runPassiveCheckForMethod(method, latencyThreshold) + } + + runCheckWithMetrics(runCheck, + c.metricsContainer.LatencyCheckRequests.WithLabelValues(c.upstreamConfig.ID, c.upstreamConfig.HTTPURL), + c.metricsContainer.LatencyCheckDuration.WithLabelValues(c.upstreamConfig.ID, c.upstreamConfig.HTTPURL)) + }(method, latencyThreshold) + } +} + +// Returns the LatencyStats instance corresponding to the specified RPC method. +// This method is thread-safe. +func (c *LatencyCheck) getLatencyCircuitBreaker(method string) LatencyCircuitBreaker { + c.lock.Lock() + defer c.lock.Unlock() + + stats, exists := c.methodLatencyBreaker[method] + + if !exists { + // This is the first time we are checking this method so initialize its LatencyStats instance. + stats = NewLatencyStats(c.routingConfig, method) + c.methodLatencyBreaker[method] = stats + } + + return stats +} + +// This method runs the passive latency check for the specified method and latency threshold. +func (c *LatencyCheck) runPassiveCheckForMethod(method string, latencyThreshold time.Duration) { + ctx, cancel := context.WithTimeout(context.Background(), RPCRequestTimeout) + defer cancel() + + latencyBreaker := c.getLatencyCircuitBreaker(method) + + // Make and record the request. + var duration time.Duration + duration, c.Err = c.client.RecordLatency(ctx, method) + // TODO(polsar): The error must also pass the checks specified in the config + // (i.e. match HTTP code, JSON RPC code, and error message). + // Fixing this is not a priority since we're not currently using passive health checking. + isError := c.Err != nil + c.errorCircuitBreaker.RecordRequest(isError) + latencyBreaker.RecordLatency(duration) + + if isError { + c.metricsContainer.LatencyCheckErrors.WithLabelValues( + c.upstreamConfig.ID, + c.upstreamConfig.HTTPURL, + metrics.HTTPRequest, + ).Inc() + } else if duration >= latencyThreshold { + c.metricsContainer.LatencyCheckHighLatencies.WithLabelValues( + c.upstreamConfig.ID, + c.upstreamConfig.HTTPURL, + metrics.HTTPRequest, + ).Inc() + } + + c.metricsContainer.Latency.WithLabelValues( + c.upstreamConfig.ID, + c.upstreamConfig.HTTPURL, + ).Set(float64(duration.Milliseconds())) + + c.logger.Debug("Ran passive LatencyCheck.", zap.Any("upstreamID", c.upstreamConfig.ID), zap.Any("latency", duration), zap.Error(c.Err)) +} + +func (c *LatencyCheck) GetUnhealthyReason(methods []string) conf.UnhealthyReason { + if c.errorCircuitBreaker.IsOpen() { + c.logger.Debug( + "LatencyCheck is not passing due to too many errors.", + zap.String("upstreamID", c.upstreamConfig.ID), + zap.Error(c.Err), + ) + + return conf.ReasonErrorRate + } + + c.lock.Lock() + defer c.lock.Unlock() + + // Only consider the passed methods, even if other methods' circuit breakers might be open. + // + // TODO(polsar): If the circuit breaker for any of the passed methods is open, we consider this upstream + // as unhealthy for all the other passed methods, even if their circuit breakers are closed. This might + // be undesirable, though since all the methods are part of the same request, we would have to somehow + // modify the request to only contain the healthy methods. This seems like more trouble than is worth. + for _, method := range methods { + if breaker, exists := c.methodLatencyBreaker[method]; exists && breaker.IsOpen() { + c.logger.Debug( + "LatencyCheck is not passing due to high latency of an RPC method.", + zap.String("upstreamID", c.upstreamConfig.ID), + zap.Any("method", method), + zap.Error(c.Err), + ) + + return conf.ReasonLatencyTooHighRate + } + } + + return conf.ReasonUnknownOrHealthy +} + +func (c *LatencyCheck) RecordRequest(data *types.RequestData) { + if c.routingConfig.PassiveLatencyChecking { + return + } + + latencyCircuitBreaker := c.getLatencyCircuitBreaker(data.Method) + latencyCircuitBreaker.RecordLatency(data.Latency) + + if data.Latency >= latencyCircuitBreaker.GetThreshold() { + c.metricsContainer.LatencyCheckHighLatencies.WithLabelValues( + c.upstreamConfig.ID, + c.upstreamConfig.HTTPURL, + metrics.HTTPRequest, + ).Inc() + } + + if data.HTTPResponseCode >= http.StatusBadRequest { + // No RPC responses are available since the HTTP request errored out. + // TODO(polsar): We might want to emit a Prometheus stat like we do for an RPC error below. + c.errorCircuitBreaker.RecordRequest(c.isError( + strconv.Itoa(data.HTTPResponseCode), + "", + "", + )) // HTTP request error + } else if data.ResponseBody != nil { + for _, resp := range data.ResponseBody.GetSubResponses() { + if resp.Error != nil { + // Do not ignore this response even if it does not correspond to an RPC request. + if c.isError("", strconv.Itoa(resp.Error.Code), resp.Error.Message) { + c.metricsContainer.LatencyCheckErrors.WithLabelValues( + c.upstreamConfig.ID, + c.upstreamConfig.HTTPURL, + metrics.HTTPRequest, + ).Inc() + + // Even though this is a single HTTP request, we count each RPC JSON subresponse error. + c.errorCircuitBreaker.RecordRequest(true) // JSON RPC subrequest error + } else { + c.errorCircuitBreaker.RecordRequest(false) // JSON RPC subrequest OK + } + } + } + + c.errorCircuitBreaker.RecordRequest(false) // HTTP request OK + } + // TODO(polsar): What does it mean when `data.ResponseBody == nil` and no HTTP error occurred? + // Log this strange case as an error. + + c.metricsContainer.Latency.WithLabelValues( + c.upstreamConfig.ID, + c.upstreamConfig.HTTPURL, + ).Set(float64(data.Latency.Milliseconds())) +} + +func (c *LatencyCheck) isError(httpCode, jsonRPCCode, errorMsg string) bool { + if isMatchForPatterns(httpCode, c.routingConfig.Errors.HTTPCodes) || + isMatchForPatterns(jsonRPCCode, c.routingConfig.Errors.JSONRPCCodes) || + isErrorMatches(errorMsg, c.routingConfig.Errors.ErrorStrings) { + return true + } + + return false +} + +func isMatchForPatterns(responseCode string, patterns []string) bool { + if responseCode == "" { + return false + } + + if len(patterns) == 0 { + return true + } + + for _, pattern := range patterns { + if isMatch(responseCode, pattern) { + return true + } + } + + return false +} + +// Returns true iff the response code matches the pattern using ResponseCodeWildcard as the wildcard character. +func isMatch(responseCode, pattern string) bool { + if len(responseCode) != len(pattern) { + return false + } + + for i, x := range responseCode { + // TODO(polsar): Unicode sucks. Fix this awkward conversion voodoo. + y := string(pattern[i]) + + if strings.EqualFold(y, string(ResponseCodeWildcard)) { + continue + } + + if string(x) != y { + return false + } + } + + return true +} + +func isErrorMatches(errorMsg string, errors []string) bool { + if errorMsg == "" { + return false + } + + if len(errors) == 0 { + return true + } + + for _, errorSubstr := range errors { + // TODO(polsar): Add support for regular expression matching. + if strings.Contains(errorMsg, errorSubstr) { + return true + } + } + + return false +} + +func boolToInt(b bool) int { + if b { + return 1 + } + + return 0 +} + +func getDetectionWindow(routingConfig *conf.RoutingConfig) time.Duration { + if routingConfig != nil && routingConfig.DetectionWindow != nil { + return *routingConfig.DetectionWindow + } + + return conf.DefaultDetectionWindow +} + +func getBanWindow(routingConfig *conf.RoutingConfig) time.Duration { + if routingConfig != nil && routingConfig.BanWindow != nil { + return *routingConfig.BanWindow + } + + return conf.DefaultBanWindow +} + +func getLatencyThreshold(routingConfig *conf.RoutingConfig, method string) time.Duration { + if routingConfig != nil && routingConfig.Latency != nil { + if latency, exists := routingConfig.Latency.MethodLatencyThresholds[method]; exists { + return latency + } + } + + return conf.DefaultMaxLatency +} + +func getErrorsRate(routingConfig *conf.RoutingConfig) float64 { + if routingConfig != nil && routingConfig.Errors != nil { + return routingConfig.Errors.Rate + } + + return conf.DefaultErrorRate +} diff --git a/internal/checks/latency_test.go b/internal/checks/latency_test.go new file mode 100644 index 00000000..8578fec2 --- /dev/null +++ b/internal/checks/latency_test.go @@ -0,0 +1,138 @@ +package checks + +import ( + "testing" + "time" + + "github.com/satsuma-data/node-gateway/internal/client" + "github.com/satsuma-data/node-gateway/internal/config" + "github.com/satsuma-data/node-gateway/internal/metrics" + "github.com/satsuma-data/node-gateway/internal/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.uber.org/zap" +) + +func helperTestLatencyChecker(t *testing.T, latency1, latency2 time.Duration, reason config.UnhealthyReason) { + t.Helper() + + methods := []string{"eth_call", "eth_getLogs"} + + ethClient := mocks.NewEthClient(t) + ethClient.EXPECT().RecordLatency(mock.Anything, methods[0]).Return(latency1, nil) + ethClient.EXPECT().RecordLatency(mock.Anything, methods[1]).Return(latency2, nil) + + mockEthClientGetter := func(url string, credentials *config.BasicAuthConfig, additionalRequestHeaders *[]config.RequestHeaderConfig) (client.EthClient, error) { //nolint:nolintlint,revive // Legacy + return ethClient, nil + } + + checker := NewLatencyChecker( + defaultUpstreamConfig, + defaultRoutingConfig, + mockEthClientGetter, + metrics.NewContainer(config.TestChainName), + zap.L(), + ) + + assert.Equal(t, reason, checker.GetUnhealthyReason(methods)) + + ethClient.AssertNumberOfCalls(t, "RecordLatency", 2) +} + +func TestLatencyChecker_TwoMethods_BothLatenciesLessThanThreshold(t *testing.T) { + helperTestLatencyChecker(t, 2*time.Millisecond, 3*time.Millisecond, config.ReasonUnknownOrHealthy) +} + +func TestLatencyChecker_TwoMethods_BothLatenciesJustBelowThreshold(t *testing.T) { + helperTestLatencyChecker(t, (10000-1)*time.Millisecond, (2000-1)*time.Millisecond, config.ReasonUnknownOrHealthy) +} + +func TestLatencyChecker_TwoMethods_FirstLatencyTooHigh(t *testing.T) { + helperTestLatencyChecker(t, 10000*time.Millisecond, (2000-1)*time.Millisecond, config.ReasonLatencyTooHighRate) +} + +func TestLatencyChecker_TwoMethods_SecondLatencyTooHigh(t *testing.T) { + helperTestLatencyChecker(t, (10000-1)*time.Millisecond, 2000*time.Millisecond, config.ReasonLatencyTooHighRate) +} + +func TestLatencyChecker_TwoMethods_BothLatenciesTooHigh(t *testing.T) { + helperTestLatencyChecker(t, 10002*time.Millisecond, 2003*time.Millisecond, config.ReasonLatencyTooHighRate) +} + +func Test_isMatchForPatterns_True(t *testing.T) { + Assert := assert.New(t) + + Assert.True(isMatchForPatterns("400", []string{})) + + Assert.True(isMatchForPatterns("400", []string{"400"})) + Assert.True(isMatchForPatterns("400", []string{"4XX"})) + Assert.True(isMatchForPatterns("400", []string{"X00"})) + Assert.True(isMatchForPatterns("400", []string{"400", "500"})) + Assert.True(isMatchForPatterns("500", []string{"400", "500"})) +} + +func Test_isMatchForPatterns_False(t *testing.T) { + Assert := assert.New(t) + + Assert.False(isMatchForPatterns("", []string{""})) + Assert.False(isMatchForPatterns("", []string{"400"})) + + Assert.False(isMatchForPatterns("400", []string{"500"})) + Assert.False(isMatchForPatterns("400", []string{"4X1"})) + Assert.False(isMatchForPatterns("410", []string{"X00"})) + Assert.False(isMatchForPatterns("400", []string{"401", "5X0"})) + Assert.False(isMatchForPatterns("503", []string{"400", "500"})) +} + +func Test_isMatch_True(t *testing.T) { + Assert := assert.New(t) + + Assert.True(isMatch("400", "400")) + Assert.True(isMatch("400", "4x0")) + Assert.True(isMatch("400", "40X")) + Assert.True(isMatch("400", "4Xx")) + Assert.True(isMatch("400", "XxX")) + + Assert.True(isMatch("38765", "38XXX")) + Assert.True(isMatch("38765", "XX765")) +} + +func Test_isMatch_False(t *testing.T) { + Assert := assert.New(t) + + Assert.False(isMatch("400", "40")) + Assert.False(isMatch("40", "400")) + + Assert.False(isMatch("400", "500")) + Assert.False(isMatch("400", "4x2")) + Assert.False(isMatch("400", "41X")) + Assert.False(isMatch("400", "6Xx")) + Assert.False(isMatch("400", "X7X")) +} + +func Test_isErrorMatches_True(t *testing.T) { + Assert := assert.New(t) + + Assert.True(isErrorMatches("a", []string{})) + Assert.True(isErrorMatches("a", []string{"a"})) + Assert.True(isErrorMatches("aa", []string{"a"})) + Assert.True(isErrorMatches("a", []string{"a", "b"})) + Assert.True(isErrorMatches("aa", []string{"a", "b"})) + Assert.True(isErrorMatches("some error", []string{"a", "err"})) + Assert.True(isErrorMatches("error string", []string{"error string"})) + Assert.True(isErrorMatches("prefix error string suffix", []string{"error string"})) + + Assert.True(isErrorMatches("aba", []string{"ab"})) + Assert.True(isErrorMatches("aba", []string{"ba"})) +} + +func Test_isErrorMatches_False(t *testing.T) { + Assert := assert.New(t) + + Assert.False(isErrorMatches("", []string{})) + Assert.False(isErrorMatches("", []string{"a", "b"})) + + Assert.False(isErrorMatches("b", []string{"a"})) + Assert.False(isErrorMatches("a", []string{"aa"})) + Assert.False(isErrorMatches("aa", []string{"aba"})) +} diff --git a/internal/checks/manager.go b/internal/checks/manager.go index 5a2f4c26..474c075f 100644 --- a/internal/checks/manager.go +++ b/internal/checks/manager.go @@ -31,18 +31,44 @@ type HealthCheckManager interface { StartHealthChecks() IsInitialized() bool GetUpstreamStatus(upstreamID string) *types.UpstreamStatus + RecordRequest(upstreamID string, data *types.RequestData) } type healthCheckManager struct { - upstreamIDToStatus map[string]*types.UpstreamStatus - ethClientGetter client.EthClientGetter - newBlockHeightCheck func(*conf.UpstreamConfig, client.EthClientGetter, BlockHeightObserver, *metrics.Container, *zap.Logger) types.BlockHeightChecker - newPeerCheck func(*conf.UpstreamConfig, client.EthClientGetter, *metrics.Container, *zap.Logger) types.Checker - newSyncingCheck func(*conf.UpstreamConfig, client.EthClientGetter, *metrics.Container, *zap.Logger) types.Checker blockHeightObserver BlockHeightObserver + newPeerCheck func( + *conf.UpstreamConfig, + client.EthClientGetter, + *metrics.Container, + *zap.Logger, + ) types.Checker + newBlockHeightCheck func( + *conf.UpstreamConfig, + client.EthClientGetter, + BlockHeightObserver, + *metrics.Container, + *zap.Logger, + ) types.BlockHeightChecker + upstreamIDToStatus map[string]*types.UpstreamStatus + newSyncingCheck func( + *conf.UpstreamConfig, + client.EthClientGetter, + *metrics.Container, + *zap.Logger, + ) types.Checker + newLatencyCheck func( + *conf.UpstreamConfig, + *conf.RoutingConfig, + client.EthClientGetter, + *metrics.Container, + *zap.Logger, + ) types.LatencyChecker + ethClientGetter client.EthClientGetter healthCheckTicker *time.Ticker metricsContainer *metrics.Container logger *zap.Logger + globalRoutingConfig conf.RoutingConfig + routingConfig conf.RoutingConfig configs []conf.UpstreamConfig isInitialized atomic.Bool } @@ -50,6 +76,8 @@ type healthCheckManager struct { func NewHealthCheckManager( ethClientGetter client.EthClientGetter, config []conf.UpstreamConfig, + routingConfig conf.RoutingConfig, + globalRoutingConfig conf.RoutingConfig, blockHeightObserver BlockHeightObserver, healthCheckTicker *time.Ticker, metricsContainer *metrics.Container, @@ -59,9 +87,12 @@ func NewHealthCheckManager( upstreamIDToStatus: make(map[string]*types.UpstreamStatus), ethClientGetter: ethClientGetter, configs: config, + routingConfig: routingConfig, + globalRoutingConfig: globalRoutingConfig, newBlockHeightCheck: NewBlockHeightChecker, newPeerCheck: NewPeerChecker, newSyncingCheck: NewSyncingChecker, + newLatencyCheck: NewLatencyChecker, blockHeightObserver: blockHeightObserver, healthCheckTicker: healthCheckTicker, metricsContainer: metricsContainer, @@ -87,6 +118,10 @@ func (h *healthCheckManager) GetUpstreamStatus(upstreamID string) *types.Upstrea panic(fmt.Sprintf("Upstream ID %s not found!", upstreamID)) } +func (h *healthCheckManager) RecordRequest(upstreamID string, data *types.RequestData) { + h.GetUpstreamStatus(upstreamID).LatencyCheck.RecordRequest(data) +} + func (h *healthCheckManager) setUpstreamStatus(upstreamID string, status *types.UpstreamStatus) { h.upstreamIDToStatus[upstreamID] = status } @@ -114,7 +149,13 @@ func (h *healthCheckManager) initializeChecks() { go func() { defer innerWG.Done() - blockHeightCheck = h.newBlockHeightCheck(&config, client.NewEthClient, h.blockHeightObserver, h.metricsContainer, h.logger) + blockHeightCheck = h.newBlockHeightCheck( + &config, + client.NewEthClient, + h.blockHeightObserver, + h.metricsContainer, + h.logger, + ) }() var peerCheck types.Checker @@ -124,7 +165,12 @@ func (h *healthCheckManager) initializeChecks() { go func() { defer innerWG.Done() - peerCheck = h.newPeerCheck(&config, client.NewEthClient, h.metricsContainer, h.logger) + peerCheck = h.newPeerCheck( + &config, + client.NewEthClient, + h.metricsContainer, + h.logger, + ) }() var syncingCheck types.Checker @@ -134,7 +180,28 @@ func (h *healthCheckManager) initializeChecks() { go func() { defer innerWG.Done() - syncingCheck = h.newSyncingCheck(&config, client.NewEthClient, h.metricsContainer, h.logger) + syncingCheck = h.newSyncingCheck( + &config, + client.NewEthClient, + h.metricsContainer, + h.logger, + ) + }() + + var latencyCheck types.LatencyChecker + + innerWG.Add(1) + + go func() { + defer innerWG.Done() + + latencyCheck = h.newLatencyCheck( + &config, + &h.routingConfig, + client.NewEthClient, + h.metricsContainer, + h.logger, + ) }() innerWG.Wait() @@ -146,6 +213,7 @@ func (h *healthCheckManager) initializeChecks() { BlockHeightCheck: blockHeightCheck, PeerCheck: peerCheck, SyncingCheck: syncingCheck, + LatencyCheck: latencyCheck, }) mutex.Unlock() }() @@ -189,6 +257,13 @@ func (h *healthCheckManager) runChecksOnce() { defer wg.Done() c.RunCheck() }(h.GetUpstreamStatus(config.ID).SyncingCheck) + + wg.Add(1) + + go func(c types.LatencyChecker) { + defer wg.Done() + c.RunPassiveCheck() + }(h.GetUpstreamStatus(config.ID).LatencyCheck) } wg.Wait() diff --git a/internal/checks/manager_test.go b/internal/checks/manager_test.go index 8ce6973f..36b75fde 100644 --- a/internal/checks/manager_test.go +++ b/internal/checks/manager_test.go @@ -36,13 +36,24 @@ func TestHealthCheckManager(t *testing.T) { HealthCheckConfig: config.HealthCheckConfig{UseWSForBlockHeight: new(bool)}, }, } + routingConfig := config.RoutingConfig{} + globalRoutingConfig := config.RoutingConfig{} tickerChan := make(chan time.Time) ticker := &time.Ticker{C: tickerChan} metricsContainer := metrics.NewContainer(config.TestChainName) - manager := NewHealthCheckManager(mockEthClientGetter, configs, nil, ticker, metricsContainer, zap.L()) + manager := NewHealthCheckManager( + mockEthClientGetter, + configs, + routingConfig, + globalRoutingConfig, + nil, + ticker, + metricsContainer, + zap.L(), + ) manager.(*healthCheckManager).newBlockHeightCheck = func( *config.UpstreamConfig, client.EthClientGetter, diff --git a/internal/checks/peers.go b/internal/checks/peers.go index ec36c79b..c831f4d3 100644 --- a/internal/checks/peers.go +++ b/internal/checks/peers.go @@ -69,7 +69,7 @@ func (c *PeerCheck) Initialize() error { func (c *PeerCheck) RunCheck() { if c.client == nil { if err := c.Initialize(); err != nil { - c.logger.Error("Errorr initializing PeerCheck.", zap.Any("upstreamID", c.upstreamConfig.ID), zap.Error(err)) + c.logger.Error("Error initializing PeerCheck.", zap.Any("upstreamID", c.upstreamConfig.ID), zap.Error(err)) c.metricsContainer.PeerCountCheckErrors.WithLabelValues(c.upstreamConfig.ID, c.upstreamConfig.HTTPURL, metrics.HTTPInit).Inc() } } @@ -106,6 +106,8 @@ func (c *PeerCheck) runCheck() { } func (c *PeerCheck) IsPassing() bool { + // TODO(polsar): This method is unused. Instead, the decision whether this check is passing is made here: + // https://github.com/satsuma-xyz/node-gateway/blob/b7f20aa2ad97f53772e9fa1565a300be7c0fff78/internal/route/node_filter.go#L61 if c.ShouldRun && (c.Err != nil || c.PeerCount < MinimumPeerCount) { c.logger.Debug("PeerCheck is not passing.", zap.String("upstreamID", c.upstreamConfig.ID), zap.Any("peerCount", c.PeerCount), zap.Error(c.Err)) diff --git a/internal/checks/syncing.go b/internal/checks/syncing.go index 1e40ff67..569ea408 100644 --- a/internal/checks/syncing.go +++ b/internal/checks/syncing.go @@ -16,10 +16,9 @@ type SyncingCheck struct { clientGetter client.EthClientGetter metricsContainer *metrics.Container logger *zap.Logger - - upstreamConfig *conf.UpstreamConfig - IsSyncing bool - ShouldRun bool + upstreamConfig *conf.UpstreamConfig + IsSyncing bool + ShouldRun bool } func NewSyncingChecker(upstreamConfig *conf.UpstreamConfig, clientGetter client.EthClientGetter, metricsContainer *metrics.Container, logger *zap.Logger) types.Checker { @@ -55,7 +54,7 @@ func (c *SyncingCheck) Initialize() error { c.runCheck() if isMethodNotSupportedErr(c.Err) { - c.logger.Debug("PeerCheck is not supported by upstream, not running check.", zap.Any("upstreamID", c.upstreamConfig.ID)) + c.logger.Debug("SyncingCheck is not supported by upstream, not running check.", zap.Any("upstreamID", c.upstreamConfig.ID)) c.ShouldRun = false } @@ -109,6 +108,8 @@ func (c *SyncingCheck) runCheck() { } func (c *SyncingCheck) IsPassing() bool { + // TODO(polsar): This method is unused. Instead, the decision whether this check is passing is made here: + // https://github.com/satsuma-xyz/node-gateway/blob/b7f20aa2ad97f53772e9fa1565a300be7c0fff78/internal/route/node_filter.go#L98 if c.ShouldRun && (c.IsSyncing || c.Err != nil) { c.logger.Error("SyncingCheck is not passing.", zap.String("upstreamID", c.upstreamConfig.ID), zap.Any("isSyncing", c.IsSyncing), zap.Error(c.Err)) diff --git a/internal/client/ethereum_client.go b/internal/client/ethereum_client.go index be5d0415..2a155ae5 100644 --- a/internal/client/ethereum_client.go +++ b/internal/client/ethereum_client.go @@ -24,12 +24,45 @@ type NewHeadHandler struct { OnError func(failure string) } +type Client ethclient.Client + //go:generate mockery --output ../mocks --name EthClient --with-expecter type EthClient interface { SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) PeerCount(ctx context.Context) (uint64, error) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, error) + RecordLatency(ctx context.Context, method string) (time.Duration, error) +} + +func (c *Client) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { + return (*ethclient.Client)(c).SubscribeNewHead(ctx, ch) +} + +func (c *Client) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { + return (*ethclient.Client)(c).HeaderByNumber(ctx, number) +} + +func (c *Client) PeerCount(ctx context.Context) (uint64, error) { + return (*ethclient.Client)(c).PeerCount(ctx) +} + +func (c *Client) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, error) { + return (*ethclient.Client)(c).SyncProgress(ctx) +} + +// RecordLatency calls the specified RPC method using the given context and returns the duration of the call, +// as well as the error if one occurred. No arguments are passed to the RPC method. +// +// TODO(polsar): If the given method expects one or more arguments, it will return an error that will be passed +// +// to the caller. We should detect this type of error and not return it, since the call has otherwise succeeded, +// which is the only thing the caller cares about. +func (c *Client) RecordLatency(ctx context.Context, method string) (time.Duration, error) { + start := time.Now() + err := (*ethclient.Client)(c).Client().CallContext(ctx, nil, method) + + return time.Since(start), err } type EthClientGetter func(url string, credentials *config.BasicAuthConfig, additionalRequestHeaders *[]config.RequestHeaderConfig) (EthClient, error) @@ -42,7 +75,7 @@ func NewEthClient(url string, credentials *config.BasicAuthConfig, additionalReq setAdditionalRequestHeaders(rpcClient, additionalRequestHeaders) - return ethclient.NewClient(rpcClient), nil + return (*Client)(ethclient.NewClient(rpcClient)), nil } func getRPCClientWithAuthHeader(url string, credentials *config.BasicAuthConfig) (*rpc.Client, error) { diff --git a/internal/config/config.go b/internal/config/config.go index 67e2d722..d880b40a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config //nolint:nolintlint,typecheck // Legacy import ( "errors" "os" + "slices" "strings" "time" @@ -12,12 +13,34 @@ import ( type NodeType string -const DefDetectionWindow = time.Minute -const DefBanWindow = 5 * time.Minute +const ( + DefaultBanWindow = 5 * time.Minute + DefaultDetectionWindow = time.Minute + // DefaultMaxLatency is used when the latency threshold is not specified in the config. + // TODO(polsar): We should probably use a lower value. + DefaultMaxLatency = 10 * time.Second + DefaultErrorRate = 0.25 + DefaultLatencyTooHighRate = 0.5 // TODO(polsar): Expose this parameter in the config. + Archive NodeType = "archive" + Full NodeType = "full" + // LatencyCheckMethod is a dummy method we use to measure the latency of an upstream RPC endpoint. + // https://docs.infura.io/api/networks/ethereum/json-rpc-methods/eth_chainid + LatencyCheckMethod = "eth_chainId" + // PassiveLatencyChecking indicates whether to use live (active) requests as data for the LatencyChecker (false) + // or synthetic (passive) periodic requests (true). + // TODO(polsar): This setting is currently not configurable via the YAML config file. + // TODO(polsar): We may also consider a hybrid request latency/error checking using both active and passive requests. + PassiveLatencyChecking = false +) + +// UnhealthyReason is the reason why a health check failed. We use it to select the most appropriate upstream to route to +// if all upstreams are unhealthy and the `alwaysRoute` option is true. +type UnhealthyReason int const ( - Archive NodeType = "archive" - Full NodeType = "full" + ReasonUnknownOrHealthy = iota + ReasonErrorRate + ReasonLatencyTooHighRate ) type UpstreamConfig struct { @@ -30,10 +53,7 @@ type UpstreamConfig struct { GroupID string `yaml:"group"` NodeType NodeType `yaml:"nodeType"` RequestHeadersConfig []RequestHeaderConfig `yaml:"requestHeaders"` -} - -func newDuration(d time.Duration) *time.Duration { - return &d + HealthStatus UnhealthyReason // The default value of this field is 0 (ReasonUnknownOrHealthy). } func (c *UpstreamConfig) isValid(groups []GroupConfig) bool { @@ -174,19 +194,21 @@ func IsGroupsValid(groups []GroupConfig) bool { } type GlobalConfig struct { - Routing RoutingConfig `yaml:"routing"` Cache CacheConfig `yaml:"cache"` + Routing RoutingConfig `yaml:"routing"` Port int `yaml:"port"` } func (c *GlobalConfig) setDefaults() { - c.Routing.setDefaults() + c.Routing.setDefaults(nil, false) } type CacheConfig struct { Redis string `yaml:"redis"` } +// ErrorsConfig +// TODO(polsar): Add the minimum number of requests in the detection window required to apply the error rate. type ErrorsConfig struct { HTTPCodes []string `yaml:"httpCodes"` JSONRPCCodes []string `yaml:"jsonRpcCodes"` @@ -194,42 +216,255 @@ type ErrorsConfig struct { Rate float64 `yaml:"rate"` } +func (c *ErrorsConfig) merge(globalConfig *ErrorsConfig) { + if globalConfig == nil { + return + } + + // TODO(polsar): Can we somehow combine these three sections into one to avoid code duplication? + c.HTTPCodes = append(c.HTTPCodes, globalConfig.HTTPCodes...) + c.HTTPCodes = sortAndRemoveDuplicates(c.HTTPCodes) + + c.JSONRPCCodes = append(c.JSONRPCCodes, globalConfig.JSONRPCCodes...) + c.JSONRPCCodes = sortAndRemoveDuplicates(c.JSONRPCCodes) + + c.ErrorStrings = append(c.ErrorStrings, globalConfig.ErrorStrings...) + c.ErrorStrings = sortAndRemoveDuplicates(c.ErrorStrings) +} + +func (c *ErrorsConfig) initialize(globalConfig *RoutingConfig) { + var globalErrorsConfig *ErrorsConfig + if globalConfig != nil { + globalErrorsConfig = globalConfig.Errors + } + + if c.Rate == 0 { + if globalErrorsConfig == nil { + c.Rate = DefaultErrorRate + } else { + c.Rate = globalErrorsConfig.Rate + } + } +} + +// Sorts in-place and removes duplicates from the specified slice. +func sortAndRemoveDuplicates(s []string) []string { + slices.Sort(s) + return slices.Compact(s) +} + type MethodConfig struct { Name string `yaml:"method"` Threshold time.Duration `yaml:"threshold"` } +func (c *MethodConfig) isMethodConfigValid(passiveLatencyChecking bool) bool { + if c == nil { + return true + } + + if passiveLatencyChecking { + // TODO(polsar): Validate the method name: https://ethereum.org/en/developers/docs/apis/json-rpc/ + return !strings.EqualFold(c.Name, LatencyCheckMethod) + } + + return true +} + +// LatencyConfig +// TODO(polsar): Add the minimum number of latencies in the detection window required to apply the threshold. +// TODO(polsar): Add other aggregation options. Currently, the average of latencies in the detection windows is used. type LatencyConfig struct { + MethodLatencyThresholds map[string]time.Duration + Methods []MethodConfig `yaml:"methods"` Threshold time.Duration `yaml:"threshold"` } +func (c *LatencyConfig) merge(globalConfig *LatencyConfig) { + if globalConfig == nil { + return + } + + for method, latencyThreshold := range globalConfig.MethodLatencyThresholds { + if _, exists := c.MethodLatencyThresholds[method]; !exists { + c.MethodLatencyThresholds[method] = latencyThreshold + } + } +} + +func (c *LatencyConfig) getLatencyThreshold(globalConfig *LatencyConfig) time.Duration { + if c.Threshold <= time.Duration(0) { + // The latency threshold is not configured or invalid, so use the global config's value or the default. + if globalConfig != nil { + return globalConfig.getLatencyThreshold(nil) + } + + return DefaultMaxLatency + } + + return c.Threshold +} + +func (c *LatencyConfig) getLatencyThresholdForMethod(method string, globalConfig *LatencyConfig) time.Duration { + latency, exists := c.MethodLatencyThresholds[method] + if !exists { + // Use the global config's latency value or the default. + if globalConfig != nil { + return globalConfig.getLatencyThresholdForMethod(method, nil) + } + + return DefaultMaxLatency + } + + return latency +} + +func (c *LatencyConfig) initialize(globalConfig *RoutingConfig) { + c.MethodLatencyThresholds = make(map[string]time.Duration) + + if c.Methods == nil { + return + } + + var globalLatencyConfig *LatencyConfig + if globalConfig != nil { + globalLatencyConfig = globalConfig.Latency + } + + for _, method := range c.Methods { + var threshold time.Duration + + if method.Threshold <= time.Duration(0) { + // The method's latency threshold is not configured or invalid. + if c.Threshold <= time.Duration(0) && globalLatencyConfig != nil { + // Use the top-level value. + threshold = globalLatencyConfig.getLatencyThresholdForMethod(method.Name, nil) + } else { + // Use the global config latency value for the method. + threshold = c.getLatencyThreshold(globalLatencyConfig) + } + } else { + threshold = method.Threshold + } + + c.MethodLatencyThresholds[method.Name] = threshold + } +} + +func (c *LatencyConfig) isLatencyConfigValid(passiveLatencyChecking bool) bool { + if c == nil { + return true + } + + for _, method := range c.Methods { + if !method.isMethodConfigValid(passiveLatencyChecking) { + return false + } + } + + return true +} + type RoutingConfig struct { - AlwaysRoute *bool `yaml:"alwaysRoute"` - Errors *ErrorsConfig `yaml:"errors"` - Latency *LatencyConfig `yaml:"latency"` - DetectionWindow *time.Duration `yaml:"detectionWindow"` - BanWindow *time.Duration `yaml:"banWindow"` - MaxBlocksBehind int `yaml:"maxBlocksBehind"` + AlwaysRoute *bool `yaml:"alwaysRoute"` + Errors *ErrorsConfig `yaml:"errors"` + Latency *LatencyConfig `yaml:"latency"` + DetectionWindow *time.Duration `yaml:"detectionWindow"` + BanWindow *time.Duration `yaml:"banWindow"` + MaxBlocksBehind int `yaml:"maxBlocksBehind"` + PassiveLatencyChecking bool + IsInitialized bool +} + +func (r *RoutingConfig) IsEnhancedRoutingControlEnabled() bool { + // TODO(polsar): This is temporary. Eventually, we want to have enhanced routing control enabled by default even if + // none of these fields are specified in the config YAML. + return r.Errors != nil || r.Latency != nil || r.DetectionWindow != nil || r.BanWindow != nil || r.AlwaysRoute != nil } -func (r *RoutingConfig) setDefaults() { - if r.Errors == nil && r.Latency == nil { +func (r *RoutingConfig) setDefaults(globalConfig *RoutingConfig, force bool) { + if r.IsInitialized { + return + } + + r.PassiveLatencyChecking = PassiveLatencyChecking + + if !force && !r.IsEnhancedRoutingControlEnabled() && (globalConfig == nil || !globalConfig.IsEnhancedRoutingControlEnabled()) { + // Routing config is not specified at either this or global level, so there is nothing to do. return } + if globalConfig != nil && !globalConfig.IsInitialized { + globalConfig.setDefaults(nil, true) + } + + // For each routing config value that is not specified, use the corresponding global config value if the global config + // is specified. Otherwise, use the default value. Note that if the global config is specified, it already has all + // defaults set. + if r.DetectionWindow == nil { - r.DetectionWindow = newDuration(DefDetectionWindow) + if globalConfig == nil { + r.DetectionWindow = NewDuration(DefaultDetectionWindow) + } else { + r.DetectionWindow = globalConfig.DetectionWindow + } } if r.BanWindow == nil { - r.BanWindow = newDuration(DefBanWindow) + if globalConfig == nil { + r.BanWindow = NewDuration(DefaultBanWindow) + } else { + r.BanWindow = globalConfig.BanWindow + } + } + + if r.AlwaysRoute == nil { + if globalConfig == nil { + r.AlwaysRoute = new(bool) // &false + } else { + r.AlwaysRoute = globalConfig.AlwaysRoute + } + } + + if r.Latency == nil { + if globalConfig == nil { + r.Latency = new(LatencyConfig) + } else { + r.Latency = globalConfig.Latency + } + } + + r.Latency.initialize(globalConfig) + + if r.Errors == nil { + if globalConfig == nil { + r.Errors = new(ErrorsConfig) + } else { + r.Errors = globalConfig.Errors + } + } + + r.Errors.initialize(globalConfig) + + if globalConfig != nil { + r.Latency.merge(globalConfig.Latency) + r.Errors.merge(globalConfig.Errors) } + + r.IsInitialized = true } func (r *RoutingConfig) isRoutingConfigValid() bool { - // TODO(polsar): Validate the HTTP and JSON RPC codes, and potentially methods as well. - return r.isErrorRateValid() + // TODO(polsar): Validate the HTTP and JSON RPC codes. + isValid := r.isErrorRateValid() + latency := r.Latency + + if latency != nil { + isValid = isValid && latency.isLatencyConfigValid(r.PassiveLatencyChecking) + } + + return isValid } func (r *RoutingConfig) isErrorRateValid() bool { @@ -247,6 +482,10 @@ func (r *RoutingConfig) isErrorRateValid() bool { return isValid } +func NewDuration(d time.Duration) *time.Duration { + return &d +} + type ChainCacheConfig struct { TTL time.Duration `yaml:"ttl"` } @@ -289,8 +528,8 @@ func (c *SingleChainConfig) isValid() bool { return isChainConfigValid } -func (c *SingleChainConfig) setDefaults() { - c.Routing.setDefaults() +func (c *SingleChainConfig) setDefaults(globalConfig *GlobalConfig) { + c.Routing.setDefaults(&globalConfig.Routing, false) } func isChainsValid(chainsConfig []SingleChainConfig) bool { @@ -306,8 +545,8 @@ func isChainsValid(chainsConfig []SingleChainConfig) bool { } type Config struct { - Global GlobalConfig Chains []SingleChainConfig + Global GlobalConfig } func (config *Config) setDefaults() { @@ -315,7 +554,7 @@ func (config *Config) setDefaults() { for chainIndex := range config.Chains { chainConfig := &config.Chains[chainIndex] - chainConfig.setDefaults() + chainConfig.setDefaults(&config.Global) } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e782431..45545140 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -321,8 +321,15 @@ func TestParseConfig_ValidConfig(t *testing.T) { } } -func getCommonChainsConfig() []SingleChainConfig { +func getCommonChainsConfig(routingConfig *RoutingConfig) []SingleChainConfig { + c := RoutingConfig{} + + if routingConfig != nil { + c = *routingConfig + } + return []SingleChainConfig{{ + Routing: c, Upstreams: []UpstreamConfig{ { ID: "alchemy-eth", @@ -387,42 +394,162 @@ func TestParseConfig_ValidConfigLatencyRouting_AllFieldsSet(t *testing.T) { t.Errorf("ParseConfig returned error: %v.", err) } + expectedLatencyConfig := LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "eth_call": 10000 * time.Millisecond, + "eth_getLogs": 2000 * time.Millisecond, + }, + Threshold: 1000 * time.Millisecond, + Methods: []MethodConfig{ + { + Name: "eth_getLogs", + Threshold: 2000 * time.Millisecond, + }, + { + Name: "eth_call", + Threshold: 10000 * time.Millisecond, + }, + }, + } + + expectedRoutingConfig := RoutingConfig{ + MaxBlocksBehind: 33, + DetectionWindow: NewDuration(10 * time.Minute), + BanWindow: NewDuration(50 * time.Minute), + Errors: &ErrorsConfig{ + Rate: 0.25, + HTTPCodes: []string{ + "420", + "5xx", + }, + JSONRPCCodes: []string{ + "32xxx", + }, + ErrorStrings: []string{ + "internal server error", + }, + }, + Latency: &expectedLatencyConfig, + AlwaysRoute: newBool(true), + IsInitialized: true, + } + + expectedRoutingChainConfig := expectedRoutingConfig + expectedRoutingChainConfig.MaxBlocksBehind = 0 + expectedConfig := Config{ Global: GlobalConfig{ - Routing: RoutingConfig{ - MaxBlocksBehind: 33, - DetectionWindow: newDuration(10 * time.Minute), - BanWindow: newDuration(50 * time.Minute), - Errors: &ErrorsConfig{ - Rate: 0.25, - HTTPCodes: []string{ - "5xx", - "420", - }, - JSONRPCCodes: []string{ - "32xxx", - }, - ErrorStrings: []string{ - "internal server error", - }, - }, - Latency: &LatencyConfig{ - Threshold: 1000 * time.Millisecond, - Methods: []MethodConfig{ - { - Name: "eth_getLogs", - Threshold: 2000 * time.Millisecond, - }, - { - Name: "eth_call", - Threshold: 10000 * time.Millisecond, - }, - }, - }, - AlwaysRoute: newBool(true), + Routing: expectedRoutingConfig, + }, + Chains: getCommonChainsConfig(&expectedRoutingChainConfig), + } + + if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { + t.Errorf("ParseConfig returned unexpected config - diff:\n%s", diff) + } +} + +func TestParseConfig_ValidConfigLatencyRouting_ErrorsConfigOverridesAndMerges(t *testing.T) { + config := ` + global: + routing: + errors: + rate: 0.28 + httpCodes: + - 5xx + - 420 + jsonRpcCodes: + - 32xxx + - 28282 + errorStrings: + - "internal server error" + - "freaking out" + + chains: + - chainName: ethereum + + groups: + - id: primary + priority: 0 + + upstreams: + - id: alchemy-eth + httpURL: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}" + group: primary + nodeType: full + + routing: + errors: + rate: 0.82 + httpCodes: + - 5xx + - 388 + - 4X4 + jsonRpcCodes: + - XXXX1 + - 28282 + errorStrings: + - "internal server error" + - "some weird error" + ` + configBytes := []byte(config) + + parsedConfig, err := parseConfig(configBytes) + + if err != nil { + t.Errorf("ParseConfig returned error: %v.", err) + } + + expectedRoutingConfig := RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Errors: &ErrorsConfig{ + Rate: 0.28, + HTTPCodes: []string{ + "5xx", + "420", + }, + JSONRPCCodes: []string{ + "32xxx", + "28282", + }, + ErrorStrings: []string{ + "internal server error", + "freaking out", }, }, - Chains: getCommonChainsConfig(), + Latency: &LatencyConfig{MethodLatencyThresholds: map[string]time.Duration{}}, + AlwaysRoute: newBool(false), + IsInitialized: true, + } + + expectedRoutingChainConfig := expectedRoutingConfig + expectedRoutingChainConfig.MaxBlocksBehind = 0 + expectedRoutingChainConfig.Errors = &ErrorsConfig{ + Rate: 0.82, + HTTPCodes: []string{ + "388", + "420", + "4X4", + "5xx", + }, + JSONRPCCodes: []string{ + "28282", + "32xxx", + "XXXX1", + }, + ErrorStrings: []string{ + "freaking out", + "internal server error", + "some weird error", + }, + } + + expectedConfig := Config{ + Global: GlobalConfig{ + Routing: expectedRoutingConfig, + }, + Chains: getCommonChainsConfig(&expectedRoutingChainConfig), } if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { @@ -458,20 +585,32 @@ func TestParseConfig_ValidConfigLatencyRouting_DefaultsForDetectionAndBanWindows t.Errorf("ParseConfig returned error: %v.", err) } + expectedLatencyConfig := LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{}, + Threshold: 1000 * time.Millisecond, + } + expectedConfig := Config{ Global: GlobalConfig{ Routing: RoutingConfig{ - DetectionWindow: newDuration(DefDetectionWindow), - BanWindow: newDuration(DefBanWindow), + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), Errors: &ErrorsConfig{ Rate: 0.25, }, - Latency: &LatencyConfig{ - Threshold: 1000 * time.Millisecond, - }, + Latency: &expectedLatencyConfig, + AlwaysRoute: newBool(false), + IsInitialized: true, }, }, - Chains: getCommonChainsConfig(), + Chains: getCommonChainsConfig(&RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + Latency: &expectedLatencyConfig, + AlwaysRoute: newBool(false), + IsInitialized: true, + }), } if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { @@ -501,7 +640,7 @@ func TestParseConfig_ValidConfigLatencyRouting_DefaultsForDetectionAndBanWindows } expectedConfig := Config{ - Chains: getCommonChainsConfig(), + Chains: getCommonChainsConfig(nil), } if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { @@ -561,6 +700,390 @@ func TestParseConfig_InvalidConfigLatencyRouting_InvalidRateInGlobalConfig(t *te } } +func TestParseConfig_ValidConfigLatencyRouting_MethodLatencies_TopLevelLatencySpecifiedBothPerChainAndGlobal(t *testing.T) { + config := ` + global: + routing: + latency: + threshold: 1000ms + methods: + - method: getLogs + threshold: 2000ms + - method: eth_getStorageAt + + chains: + - chainName: ethereum + routing: + latency: + threshold: 4000ms + methods: + - method: getLogs + - method: eth_getStorageAt + threshold: 6000ms + groups: + - id: primary + priority: 0 + upstreams: + - id: alchemy-eth + httpURL: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}" + group: primary + nodeType: full + ` + configBytes := []byte(config) + + parsedConfig, err := parseConfig(configBytes) + + if err != nil { + t.Errorf("ParseConfig returned error: %v.", err) + } + + expectedConfig := Config{ + Global: GlobalConfig{ + Routing: RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 2000 * time.Millisecond, + "eth_getStorageAt": 1000 * time.Millisecond, // Top-level latency default + }, + Threshold: 1000 * time.Millisecond, + Methods: []MethodConfig{ + { + Name: "getLogs", + Threshold: 2000 * time.Millisecond, + }, + { + Name: "eth_getStorageAt", + }, + }, + }, + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + AlwaysRoute: newBool(false), + IsInitialized: true, + }, + }, + + Chains: getCommonChainsConfig(&RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 4000 * time.Millisecond, // Top-level latency default + "eth_getStorageAt": 6000 * time.Millisecond, + }, + Threshold: 4000 * time.Millisecond, + Methods: []MethodConfig{ + { + Name: "getLogs", + }, + { + Name: "eth_getStorageAt", + Threshold: 6000 * time.Millisecond, + }, + }, + }, + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + AlwaysRoute: newBool(false), + IsInitialized: true, + }), + } + + if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { + t.Errorf("ParseConfig returned unexpected config - diff:\n%s", diff) + } +} + +func TestParseConfig_ValidConfigLatencyRouting_MethodLatencies_TopLevelLatencyNotSpecifiedGlobalConfig(t *testing.T) { + config := ` + global: + routing: + latency: + methods: + - method: getLogs + threshold: 2000ms + - method: eth_getStorageAt + + chains: + - chainName: ethereum + groups: + - id: primary + priority: 0 + upstreams: + - id: alchemy-eth + httpURL: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}" + group: primary + nodeType: full + ` + configBytes := []byte(config) + + parsedConfig, err := parseConfig(configBytes) + + if err != nil { + t.Errorf("ParseConfig returned error: %v.", err) + } + + expectedLatencyConfig := LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 2000 * time.Millisecond, + "eth_getStorageAt": DefaultMaxLatency, // Global default + }, + Methods: []MethodConfig{ + { + Name: "getLogs", + Threshold: 2000 * time.Millisecond, + }, + { + Name: "eth_getStorageAt", + }, + }, + } + + expectedConfig := Config{ + Global: GlobalConfig{ + Routing: RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &expectedLatencyConfig, + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + AlwaysRoute: newBool(false), + IsInitialized: true, + }, + }, + + Chains: getCommonChainsConfig(&RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &expectedLatencyConfig, + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + AlwaysRoute: newBool(false), + IsInitialized: true, + }), + } + + if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { + t.Errorf("ParseConfig returned unexpected config - diff:\n%s", diff) + } +} + +func TestParseConfig_ValidConfigLatencyRouting_MethodLatencies_TopLevelLatencySpecifiedInGlobalConfigOnly(t *testing.T) { + config := ` + global: + routing: + latency: + threshold: 1000ms + methods: + - method: getLogs + threshold: 2000ms + - method: eth_getStorageAt + + chains: + - chainName: ethereum + routing: + latency: + methods: + - method: getLogs + - method: eth_getStorageAt + threshold: 6000ms + groups: + - id: primary + priority: 0 + upstreams: + - id: alchemy-eth + httpURL: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}" + group: primary + nodeType: full + ` + configBytes := []byte(config) + + parsedConfig, err := parseConfig(configBytes) + + if err != nil { + t.Errorf("ParseConfig returned error: %v.", err) + } + + expectedConfig := Config{ + Global: GlobalConfig{ + Routing: RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 2000 * time.Millisecond, + "eth_getStorageAt": 1000 * time.Millisecond, // Top-level latency default + }, + Threshold: 1000 * time.Millisecond, + Methods: []MethodConfig{ + { + Name: "getLogs", + Threshold: 2000 * time.Millisecond, + }, + { + Name: "eth_getStorageAt", + }, + }, + }, + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + AlwaysRoute: newBool(false), + IsInitialized: true, + }, + }, + + Chains: getCommonChainsConfig(&RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 2000 * time.Millisecond, // Top-level latency for method + "eth_getStorageAt": 6000 * time.Millisecond, + }, + Methods: []MethodConfig{ + { + Name: "getLogs", + }, + { + Name: "eth_getStorageAt", + Threshold: 6000 * time.Millisecond, + }, + }, + }, + Errors: &ErrorsConfig{Rate: DefaultErrorRate}, + AlwaysRoute: newBool(false), + IsInitialized: true, + }), + } + + if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { + t.Errorf("ParseConfig returned unexpected config - diff:\n%s", diff) + } +} + +func TestParseConfig_ValidConfigLatencyRouting_MethodLatencies_TopLevelLatencyNotSpecifiedNeitherGlobalNotChain(t *testing.T) { + config := ` + global: + routing: + latency: + methods: + - method: getLogs + threshold: 2000ms + - method: eth_getStorageAt + - method: eth_chainId + threshold: 20ms + errors: + rate: 0.88 + httpCodes: + - 5xx + - 420 + jsonRpcCodes: + - 32xxx + errorStrings: + - "internal server error" + + chains: + - chainName: ethereum + routing: + latency: + methods: + - method: getLogs + - method: eth_getStorageAt + threshold: 6000ms + - method: eth_doesSomethingImportant + groups: + - id: primary + priority: 0 + upstreams: + - id: alchemy-eth + httpURL: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_API_KEY}" + group: primary + nodeType: full + ` + configBytes := []byte(config) + + parsedConfig, err := parseConfig(configBytes) + + if err != nil { + t.Errorf("ParseConfig returned error: %v.", err) + } + + expectedErrorsConfig := ErrorsConfig{ + Rate: 0.88, + HTTPCodes: []string{ + "420", + "5xx", + }, + JSONRPCCodes: []string{ + "32xxx", + }, + ErrorStrings: []string{ + "internal server error", + }, + } + + expectedConfig := Config{ + Global: GlobalConfig{ + Routing: RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 2000 * time.Millisecond, + "eth_getStorageAt": DefaultMaxLatency, // Top-level latency default + "eth_chainId": 20 * time.Millisecond, + }, + Methods: []MethodConfig{ + { + Name: "getLogs", + Threshold: 2000 * time.Millisecond, + }, + { + Name: "eth_getStorageAt", + }, + { + Name: "eth_chainId", + Threshold: 20 * time.Millisecond, + }, + }, + }, + Errors: &expectedErrorsConfig, + AlwaysRoute: newBool(false), + IsInitialized: true, + }, + }, + + Chains: getCommonChainsConfig(&RoutingConfig{ + DetectionWindow: NewDuration(DefaultDetectionWindow), + BanWindow: NewDuration(DefaultBanWindow), + Latency: &LatencyConfig{ + MethodLatencyThresholds: map[string]time.Duration{ + "getLogs": 2000 * time.Millisecond, // Top-level latency for method + "eth_getStorageAt": 6000 * time.Millisecond, + "eth_doesSomethingImportant": DefaultMaxLatency, + "eth_chainId": 20 * time.Millisecond, // Inherited from global latency config + }, + Methods: []MethodConfig{ + { + Name: "getLogs", + }, + { + Name: "eth_getStorageAt", + Threshold: 6000 * time.Millisecond, + }, + { + Name: "eth_doesSomethingImportant", + }, + }, + }, + Errors: &expectedErrorsConfig, + AlwaysRoute: newBool(false), + IsInitialized: true, + }), + } + + if diff := cmp.Diff(expectedConfig, parsedConfig); diff != "" { + t.Errorf("ParseConfig returned unexpected config - diff:\n%s", diff) + } +} + func TestParseConfig_InvalidYaml(t *testing.T) { config := ` global: diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 1beb8869..2eaeb102 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -234,6 +234,57 @@ var ( }, []string{"chain_name", "upstream_id", "url", "errorType"}, ) + + latencyStatus = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: "healthcheck", + Name: "latency", + Help: "Latency of upstream.", + }, + []string{"chain_name", "upstream_id", "url"}, + ) + + latencyStatusCheckRequests = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: "healthcheck", + Name: "latency_check_requests", + Help: "Total latency check requests made.", + }, + []string{"chain_name", "upstream_id", "url"}, + ) + + latencyStatusCheckDuration = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: "healthcheck", + Name: "latency_check_duration_seconds", + Help: "Latency of latency check requests.", + Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 20, 40}, + }, + []string{"chain_name", "upstream_id", "url"}, + ) + + latencyStatusCheckErrors = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: "healthcheck", + Name: "latency_check_errors", + Help: "Errors when checking latency of upstream.", + }, + []string{"chain_name", "upstream_id", "url", "errorType"}, + ) + + latencyStatusHighLatencies = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: "healthcheck", + Name: "latency_check_high_latency", + Help: "Latency of upstream too high.", + }, + []string{"chain_name", "upstream_id", "url", "errorType"}, + ) ) type Container struct { @@ -260,6 +311,12 @@ type Container struct { SyncStatusCheckRequests *prometheus.CounterVec SyncStatusCheckDuration prometheus.ObserverVec SyncStatusCheckErrors *prometheus.CounterVec + + Latency *prometheus.GaugeVec + LatencyCheckRequests *prometheus.CounterVec + LatencyCheckDuration prometheus.ObserverVec + LatencyCheckErrors *prometheus.CounterVec + LatencyCheckHighLatencies *prometheus.CounterVec } func NewContainer(chainName string) *Container { @@ -292,6 +349,12 @@ func NewContainer(chainName string) *Container { result.SyncStatusCheckDuration = syncStatusCheckDuration.MustCurryWith(presetLabels) result.SyncStatusCheckErrors = syncStatusCheckErrors.MustCurryWith(presetLabels) + result.Latency = latencyStatus.MustCurryWith(presetLabels) + result.LatencyCheckRequests = latencyStatusCheckRequests.MustCurryWith(presetLabels) + result.LatencyCheckDuration = latencyStatusCheckDuration.MustCurryWith(presetLabels) + result.LatencyCheckErrors = latencyStatusCheckErrors.MustCurryWith(presetLabels) + result.LatencyCheckHighLatencies = latencyStatusHighLatencies.MustCurryWith(presetLabels) + return result } diff --git a/internal/mocks/EthClient.go b/internal/mocks/EthClient.go index 84805e65..834bf33c 100644 --- a/internal/mocks/EthClient.go +++ b/internal/mocks/EthClient.go @@ -11,6 +11,8 @@ import ( mock "github.com/stretchr/testify/mock" + time "time" + types "github.com/ethereum/go-ethereum/core/types" ) @@ -142,6 +144,63 @@ func (_c *EthClient_PeerCount_Call) RunAndReturn(run func(context.Context) (uint return _c } +// RecordLatency provides a mock function with given fields: ctx, method +func (_m *EthClient) RecordLatency(ctx context.Context, method string) (time.Duration, error) { + ret := _m.Called(ctx, method) + + if len(ret) == 0 { + panic("no return value specified for RecordLatency") + } + + var r0 time.Duration + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (time.Duration, error)); ok { + return rf(ctx, method) + } + if rf, ok := ret.Get(0).(func(context.Context, string) time.Duration); ok { + r0 = rf(ctx, method) + } else { + r0 = ret.Get(0).(time.Duration) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, method) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EthClient_RecordLatency_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordLatency' +type EthClient_RecordLatency_Call struct { + *mock.Call +} + +// RecordLatency is a helper method to define mock.On call +// - ctx context.Context +// - method string +func (_e *EthClient_Expecter) RecordLatency(ctx interface{}, method interface{}) *EthClient_RecordLatency_Call { + return &EthClient_RecordLatency_Call{Call: _e.mock.On("RecordLatency", ctx, method)} +} + +func (_c *EthClient_RecordLatency_Call) Run(run func(ctx context.Context, method string)) *EthClient_RecordLatency_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *EthClient_RecordLatency_Call) Return(_a0 time.Duration, _a1 error) *EthClient_RecordLatency_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *EthClient_RecordLatency_Call) RunAndReturn(run func(context.Context, string) (time.Duration, error)) *EthClient_RecordLatency_Call { + _c.Call.Return(run) + return _c +} + // SubscribeNewHead provides a mock function with given fields: ctx, ch func (_m *EthClient) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { ret := _m.Called(ctx, ch) diff --git a/internal/mocks/HealthCheckManager.go b/internal/mocks/HealthCheckManager.go index b4eb838c..797c444a 100644 --- a/internal/mocks/HealthCheckManager.go +++ b/internal/mocks/HealthCheckManager.go @@ -113,6 +113,40 @@ func (_c *HealthCheckManager_IsInitialized_Call) RunAndReturn(run func() bool) * return _c } +// RecordRequest provides a mock function with given fields: upstreamID, data +func (_m *HealthCheckManager) RecordRequest(upstreamID string, data *types.RequestData) { + _m.Called(upstreamID, data) +} + +// HealthCheckManager_RecordRequest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordRequest' +type HealthCheckManager_RecordRequest_Call struct { + *mock.Call +} + +// RecordRequest is a helper method to define mock.On call +// - upstreamID string +// - data *types.RequestData +func (_e *HealthCheckManager_Expecter) RecordRequest(upstreamID interface{}, data interface{}) *HealthCheckManager_RecordRequest_Call { + return &HealthCheckManager_RecordRequest_Call{Call: _e.mock.On("RecordRequest", upstreamID, data)} +} + +func (_c *HealthCheckManager_RecordRequest_Call) Run(run func(upstreamID string, data *types.RequestData)) *HealthCheckManager_RecordRequest_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(*types.RequestData)) + }) + return _c +} + +func (_c *HealthCheckManager_RecordRequest_Call) Return() *HealthCheckManager_RecordRequest_Call { + _c.Call.Return() + return _c +} + +func (_c *HealthCheckManager_RecordRequest_Call) RunAndReturn(run func(string, *types.RequestData)) *HealthCheckManager_RecordRequest_Call { + _c.Call.Return(run) + return _c +} + // StartHealthChecks provides a mock function with given fields: func (_m *HealthCheckManager) StartHealthChecks() { _m.Called() diff --git a/internal/mocks/LatencyChecker.go b/internal/mocks/LatencyChecker.go new file mode 100644 index 00000000..09bc101a --- /dev/null +++ b/internal/mocks/LatencyChecker.go @@ -0,0 +1,148 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + config "github.com/satsuma-data/node-gateway/internal/config" + mock "github.com/stretchr/testify/mock" + + types "github.com/satsuma-data/node-gateway/internal/types" +) + +// LatencyChecker is an autogenerated mock type for the LatencyChecker type +type LatencyChecker struct { + mock.Mock +} + +type LatencyChecker_Expecter struct { + mock *mock.Mock +} + +func (_m *LatencyChecker) EXPECT() *LatencyChecker_Expecter { + return &LatencyChecker_Expecter{mock: &_m.Mock} +} + +// GetUnhealthyReason provides a mock function with given fields: methods +func (_m *LatencyChecker) GetUnhealthyReason(methods []string) config.UnhealthyReason { + ret := _m.Called(methods) + + if len(ret) == 0 { + panic("no return value specified for GetUnhealthyReason") + } + + var r0 config.UnhealthyReason + if rf, ok := ret.Get(0).(func([]string) config.UnhealthyReason); ok { + r0 = rf(methods) + } else { + r0 = ret.Get(0).(config.UnhealthyReason) + } + + return r0 +} + +// LatencyChecker_GetUnhealthyReason_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetUnhealthyReason' +type LatencyChecker_GetUnhealthyReason_Call struct { + *mock.Call +} + +// GetUnhealthyReason is a helper method to define mock.On call +// - methods []string +func (_e *LatencyChecker_Expecter) GetUnhealthyReason(methods interface{}) *LatencyChecker_GetUnhealthyReason_Call { + return &LatencyChecker_GetUnhealthyReason_Call{Call: _e.mock.On("GetUnhealthyReason", methods)} +} + +func (_c *LatencyChecker_GetUnhealthyReason_Call) Run(run func(methods []string)) *LatencyChecker_GetUnhealthyReason_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]string)) + }) + return _c +} + +func (_c *LatencyChecker_GetUnhealthyReason_Call) Return(_a0 config.UnhealthyReason) *LatencyChecker_GetUnhealthyReason_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *LatencyChecker_GetUnhealthyReason_Call) RunAndReturn(run func([]string) config.UnhealthyReason) *LatencyChecker_GetUnhealthyReason_Call { + _c.Call.Return(run) + return _c +} + +// RecordRequest provides a mock function with given fields: data +func (_m *LatencyChecker) RecordRequest(data *types.RequestData) { + _m.Called(data) +} + +// LatencyChecker_RecordRequest_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordRequest' +type LatencyChecker_RecordRequest_Call struct { + *mock.Call +} + +// RecordRequest is a helper method to define mock.On call +// - data *types.RequestData +func (_e *LatencyChecker_Expecter) RecordRequest(data interface{}) *LatencyChecker_RecordRequest_Call { + return &LatencyChecker_RecordRequest_Call{Call: _e.mock.On("RecordRequest", data)} +} + +func (_c *LatencyChecker_RecordRequest_Call) Run(run func(data *types.RequestData)) *LatencyChecker_RecordRequest_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*types.RequestData)) + }) + return _c +} + +func (_c *LatencyChecker_RecordRequest_Call) Return() *LatencyChecker_RecordRequest_Call { + _c.Call.Return() + return _c +} + +func (_c *LatencyChecker_RecordRequest_Call) RunAndReturn(run func(*types.RequestData)) *LatencyChecker_RecordRequest_Call { + _c.Call.Return(run) + return _c +} + +// RunPassiveCheck provides a mock function with given fields: +func (_m *LatencyChecker) RunPassiveCheck() { + _m.Called() +} + +// LatencyChecker_RunPassiveCheck_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunPassiveCheck' +type LatencyChecker_RunPassiveCheck_Call struct { + *mock.Call +} + +// RunPassiveCheck is a helper method to define mock.On call +func (_e *LatencyChecker_Expecter) RunPassiveCheck() *LatencyChecker_RunPassiveCheck_Call { + return &LatencyChecker_RunPassiveCheck_Call{Call: _e.mock.On("RunPassiveCheck")} +} + +func (_c *LatencyChecker_RunPassiveCheck_Call) Run(run func()) *LatencyChecker_RunPassiveCheck_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *LatencyChecker_RunPassiveCheck_Call) Return() *LatencyChecker_RunPassiveCheck_Call { + _c.Call.Return() + return _c +} + +func (_c *LatencyChecker_RunPassiveCheck_Call) RunAndReturn(run func()) *LatencyChecker_RunPassiveCheck_Call { + _c.Call.Return(run) + return _c +} + +// NewLatencyChecker creates a new instance of LatencyChecker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewLatencyChecker(t interface { + mock.TestingT + Cleanup(func()) +}) *LatencyChecker { + mock := &LatencyChecker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/route/node_filter.go b/internal/route/node_filter.go index 53b52966..82bf48e5 100644 --- a/internal/route/node_filter.go +++ b/internal/route/node_filter.go @@ -107,6 +107,32 @@ func (f *IsDoneSyncing) Apply(_ metadata.RequestMetadata, upstreamConfig *config return true } +type IsLatencyAcceptable struct { + healthCheckManager checks.HealthCheckManager + logger *zap.Logger +} + +func (f *IsLatencyAcceptable) Apply(requestMetadata metadata.RequestMetadata, upstreamConfig *config.UpstreamConfig, _ int) bool { + upstreamStatus := f.healthCheckManager.GetUpstreamStatus(upstreamConfig.ID) + + latencyCheck, _ := upstreamStatus.LatencyCheck.(*checks.LatencyCheck) + + if latencyCheck.ShouldRun { + // TODO(polsar): If unhealthy, set the delta by which the check failed. + // For example, if the configured rate is 0.25 and the current error rate is 0.27, + // the delta is 0.27 - 0.25 = 0.02. This value can be used to rank upstreams by the + // degree to which they are unhealthy. This would help us choose the upstream to + // route to if all upstreams are unhealthy AND `alwaysRoute` config option is true. + upstreamConfig.HealthStatus = latencyCheck.GetUnhealthyReason(requestMetadata.Methods) + } + + // TODO(polsar): Note that for LatencyCheck only, we always return true. The health status + // of the upstream is instead contained in the struct's HealthStatus field. This is a bit + // clunky. Eventually, we want to change the signature of the Apply method. This will require + // significant refactoring. + return true +} + type IsCloseToGlobalMaxHeight struct { chainMetadataStore *metadata.ChainMetadataStore logger *zap.Logger @@ -267,9 +293,17 @@ func CreateSingleNodeFilter( minimumPeerCount: checks.MinimumPeerCount, } + isLatencyAcceptable := IsLatencyAcceptable{ + healthCheckManager: manager, + logger: logger, + } + return &AndFilter{ - filters: []NodeFilter{&hasEnoughPeers}, - logger: logger, + filters: []NodeFilter{ + &hasEnoughPeers, + &isLatencyAcceptable, + }, + logger: logger, } case NearGlobalMaxHeight: maxBlocksBehind := DefaultMaxBlocksBehind diff --git a/internal/route/router.go b/internal/route/router.go index 8ccd95b6..ab13bd2b 100644 --- a/internal/route/router.go +++ b/internal/route/router.go @@ -128,10 +128,18 @@ func (r *SimpleRouter) Route( start := time.Now() jsonRPCResponse, httpResponse, cached, err := r.requestExecutor.routeToConfig(ctx, requestBody, &configToRoute) - HTTPReponseCode := "" + statusCode := 0 + HTTPResponseCode := "" + r.healthCheckManager.RecordRequest(upstreamID, &types.RequestData{ + Method: requestBody.GetMethod(), + HTTPResponseCode: statusCode, + ResponseBody: jsonRPCResponse, + Latency: time.Since(start), + }) if httpResponse != nil { - HTTPReponseCode = strconv.Itoa(httpResponse.StatusCode) + statusCode = httpResponse.StatusCode + HTTPResponseCode = strconv.Itoa(statusCode) } r.metricsContainer.UpstreamRPCRequestsTotal.WithLabelValues( @@ -142,13 +150,13 @@ func (r *SimpleRouter) Route( strconv.FormatBool(cached), ).Inc() - if err != nil || httpResponse.StatusCode >= http.StatusBadRequest { + if err != nil || statusCode == 0 || statusCode >= http.StatusBadRequest { r.metricsContainer.UpstreamRPCRequestErrorsTotal.WithLabelValues( util.GetClientFromContext(ctx), upstreamID, configToRoute.HTTPURL, requestBody.GetMethod(), - HTTPReponseCode, + HTTPResponseCode, ).Inc() } @@ -194,7 +202,7 @@ func (r *SimpleRouter) Route( configToRoute.ID, configToRoute.HTTPURL, requestBody.GetMethod(), - HTTPReponseCode, + HTTPResponseCode, ).Observe(time.Since(start).Seconds()) return upstreamID, jsonRPCResponse, err diff --git a/internal/route/router_test.go b/internal/route/router_test.go index 354d55fc..4e634bfc 100644 --- a/internal/route/router_test.go +++ b/internal/route/router_test.go @@ -49,6 +49,7 @@ func TestRouter_NoHealthyUpstreams(t *testing.T) { func TestRouter_GroupUpstreamsByPriority(t *testing.T) { managerMock := mocks.NewHealthCheckManager(t) + managerMock.EXPECT().RecordRequest(mock.Anything, mock.Anything) httpClientMock := mocks.NewHTTPClient(t) httpResp := &http.Response{ @@ -124,6 +125,7 @@ func TestRouter_GroupUpstreamsByPriority(t *testing.T) { func TestGroupUpstreamsByPriority_NoGroups(t *testing.T) { managerMock := mocks.NewHealthCheckManager(t) + managerMock.EXPECT().RecordRequest(mock.Anything, mock.Anything) httpClientMock := mocks.NewHTTPClient(t) httpResp := &http.Response{ diff --git a/internal/route/routing_strategy.go b/internal/route/routing_strategy.go index 9a59c74c..3baa45c1 100644 --- a/internal/route/routing_strategy.go +++ b/internal/route/routing_strategy.go @@ -1,9 +1,11 @@ package route import ( + "slices" "sort" "sync/atomic" + conf "github.com/satsuma-data/node-gateway/internal/config" "github.com/satsuma-data/node-gateway/internal/metadata" "github.com/satsuma-data/node-gateway/internal/types" "go.uber.org/zap" @@ -12,21 +14,23 @@ import ( //go:generate mockery --output ../mocks --name RoutingStrategy --structname MockRoutingStrategy --with-expecter type RoutingStrategy interface { - // Returns the next UpstreamID a request should route to. + // RouteNextRequest returns the next UpstreamID a request should route to. RouteNextRequest( upstreamsByPriority types.PriorityToUpstreamsMap, requestMetadata metadata.RequestMetadata, ) (string, error) } type PriorityRoundRobinStrategy struct { - logger *zap.Logger - counter uint64 + logger *zap.Logger + counter uint64 + alwaysRoute bool } -func NewPriorityRoundRobinStrategy(logger *zap.Logger) *PriorityRoundRobinStrategy { +func NewPriorityRoundRobinStrategy(logger *zap.Logger, alwaysRoute bool) *PriorityRoundRobinStrategy { return &PriorityRoundRobinStrategy{ - logger: logger, - counter: 0, + logger: logger, + counter: 0, + alwaysRoute: alwaysRoute, } } @@ -44,20 +48,113 @@ func (s *PriorityRoundRobinStrategy) RouteNextRequest( upstreamsByPriority types.PriorityToUpstreamsMap, _ metadata.RequestMetadata, ) (string, error) { - prioritySorted := maps.Keys(upstreamsByPriority) + statusToUpstreamsByPriority := partitionUpstreams(upstreamsByPriority) + + var healthyUpstreamsByPriority types.PriorityToUpstreamsMap + + var exists bool + + if healthyUpstreamsByPriority, exists = statusToUpstreamsByPriority[conf.ReasonUnknownOrHealthy]; !exists { + // There are no healthy upstreams. + healthyUpstreamsByPriority = make(types.PriorityToUpstreamsMap) + } + + prioritySorted := maps.Keys(healthyUpstreamsByPriority) sort.Ints(prioritySorted) + // Note that `prioritySorted` can be empty, in which case the body of this loop will not be executed even once. for _, priority := range prioritySorted { - upstreams := upstreamsByPriority[priority] + upstreams := healthyUpstreamsByPriority[priority] if len(upstreams) > 0 { atomic.AddUint64(&s.counter, 1) - return upstreams[int(s.counter)%len(upstreams)].ID, nil + return upstreams[int(s.counter)%len(upstreams)].ID, nil //nolint:nolintlint,gosec // Legacy } s.logger.Debug("Did not find any healthy nodes in priority.", zap.Int("priority", priority)) } + // No healthy upstreams are available. If `alwaysRoute` is true, find an unhealthy upstream to route to anyway. + // TODO(polsar): At this time, the only unhealthy upstreams that can end up here are those due to high latency + // or error rate. Pass ALL configured upstreams in `upstreamsByPriority`. Their health status should be indicated + // in the UpstreamConfig.HealthStatus field. + if s.alwaysRoute { + s.logger.Warn("No healthy upstreams found but `alwaysRoute` is set to true.") + + // If available, return an upstream that's unhealthy due to high latency rate. + if upstreamsByPriorityLatencyUnhealthy, ok := statusToUpstreamsByPriority[conf.ReasonLatencyTooHighRate]; ok { + // TODO(polsar): Should we include more information in the log message? + s.logger.Info("Routing to an upstream with high latency.") + // TODO(polsar): The call can return nil, but it shouldn't be possible. Should we still check? + return getHighestPriorityUpstream(upstreamsByPriorityLatencyUnhealthy).ID, nil + } + + // If available, return an upstream that's unhealthy due to high error rate. + if upstreamsByPriorityErrorUnhealthy, ok := statusToUpstreamsByPriority[conf.ReasonErrorRate]; ok { + // TODO(polsar): Should we include more information in the log message? + s.logger.Info("Routing to an upstream with high error rate.") + // TODO(polsar): The call can return nil, but it shouldn't be possible. Should we still check? + return getHighestPriorityUpstream(upstreamsByPriorityErrorUnhealthy).ID, nil + } + + // TODO(polsar): If we get here, that means all the upstreams are unhealthy, but they are all unhealthy + // due to a reason other than high latency or error rate. We should still be able to route to one of those. + // Asana task: https://app.asana.com/0/1207397277805097/1208186611173034/f + s.logger.Error("All upstreams are unhealthy due to reasons other than high latency or error rate.") + } + + // TODO(polsar): (Once the task above is complete.) If `alwaysRoute` is true, the only way we can get here is if + // there are no upstreams in `upstreamsByPriority`. This shouldn't be possible, so we should log a critical error. return "", DefaultNoHealthyUpstreamsError } + +// Partitions the given upstreams by their health status. +func partitionUpstreams(upstreamsByPriority types.PriorityToUpstreamsMap) map[conf.UnhealthyReason]types.PriorityToUpstreamsMap { + statusToUpstreamsByPriority := make(map[conf.UnhealthyReason]types.PriorityToUpstreamsMap) + + for priority, upstreams := range upstreamsByPriority { + for _, upstream := range upstreams { + status := upstream.HealthStatus + + if upstreamsByPriorityForStatus, statusExists := statusToUpstreamsByPriority[status]; statusExists { + // The priority-to-upstreams map exists for the status. + if upstreamsForStatusAndPriority, priorityExists := upstreamsByPriorityForStatus[priority]; priorityExists { + // The upstreams slice exists for the status and priority, so append to it. + upstreamsByPriorityForStatus[priority] = append(upstreamsForStatusAndPriority, upstream) + } else { + // The upstreams slice does not exist for the status and priority, so create it. + upstreamsByPriorityForStatus[priority] = []*conf.UpstreamConfig{upstream} + } + } else { + // The priority-to-upstreams map does not exist for the status, so create it. + statusToUpstreamsByPriority[status] = types.PriorityToUpstreamsMap{ + priority: []*conf.UpstreamConfig{upstream}, + } + } + } + } + + return statusToUpstreamsByPriority +} + +// Returns the first upstream with the highest priority in the given map. Note the in our case the highest priority +// corresponds to the lowest int value. +func getHighestPriorityUpstream(upstreamsByPriority types.PriorityToUpstreamsMap) *conf.UpstreamConfig { + priorities := maps.Keys(upstreamsByPriority) + + if len(priorities) == 0 { + return nil + } + + maxPriority := slices.Min(priorities) + upstreams := upstreamsByPriority[maxPriority] + + if len(upstreams) == 0 { + // TODO(polsar): This is really an error. If a priority is a key in the passed map, + // there should be at least one upstream for it. + return nil + } + + return upstreams[0] +} diff --git a/internal/route/routing_strategy_test.go b/internal/route/routing_strategy_test.go index 96b7025d..a3db870b 100644 --- a/internal/route/routing_strategy_test.go +++ b/internal/route/routing_strategy_test.go @@ -17,7 +17,7 @@ func TestPriorityStrategy_HighPriority(t *testing.T) { 1: {cfg("erigon")}, } - strategy := NewPriorityRoundRobinStrategy(zap.L()) + strategy := NewPriorityRoundRobinStrategy(zap.L(), false) for i := 0; i < 10; i++ { firstUpstreamID, _ := strategy.RouteNextRequest(upstreams, metadata.RequestMetadata{}) @@ -40,7 +40,7 @@ func TestPriorityStrategy_LowerPriority(t *testing.T) { 1: {cfg("fallback1"), cfg("fallback2")}, } - strategy := NewPriorityRoundRobinStrategy(zap.L()) + strategy := NewPriorityRoundRobinStrategy(zap.L(), false) for i := 0; i < 10; i++ { firstUpstreamID, _ := strategy.RouteNextRequest(upstreams, metadata.RequestMetadata{}) @@ -57,7 +57,7 @@ func TestPriorityStrategy_NoUpstreams(t *testing.T) { 1: {}, } - strategy := NewPriorityRoundRobinStrategy(zap.L()) + strategy := NewPriorityRoundRobinStrategy(zap.L(), false) for i := 0; i < 10; i++ { upstreamID, err := strategy.RouteNextRequest(upstreams, metadata.RequestMetadata{}) diff --git a/internal/server/object_graph.go b/internal/server/object_graph.go index d5254a2e..f8e3f736 100644 --- a/internal/server/object_graph.go +++ b/internal/server/object_graph.go @@ -27,17 +27,38 @@ type singleChainObjectGraph struct { chainName string } -func wireSingleChainDependencies(chainConfig *config.SingleChainConfig, logger *zap.Logger, rpcCache *cache.RPCCache) singleChainObjectGraph { +func wireSingleChainDependencies( + globalConfig config.GlobalConfig, //nolint:gocritic // Legacy + chainConfig *config.SingleChainConfig, + logger *zap.Logger, + rpcCache *cache.RPCCache, +) singleChainObjectGraph { metricContainer := metrics.NewContainer(chainConfig.ChainName) chainMetadataStore := metadata.NewChainMetadataStore() ticker := time.NewTicker(checks.PeriodicHealthCheckInterval) - healthCheckManager := checks.NewHealthCheckManager(client.NewEthClient, chainConfig.Upstreams, chainMetadataStore, ticker, metricContainer, logger) + healthCheckManager := checks.NewHealthCheckManager( + client.NewEthClient, + chainConfig.Upstreams, + chainConfig.Routing, + globalConfig.Routing, + chainMetadataStore, + ticker, + metricContainer, + logger, + ) + + alwaysRoute := false + if chainConfig.Routing.AlwaysRoute != nil { + alwaysRoute = *chainConfig.Routing.AlwaysRoute + } + // TODO(polsar): Here, the HealthCheckManager is wired into the primary FilteringRoutingStrategy. + // We may need to wire it into the secondary PriorityRoundRobinStrategy as well. enabledNodeFilters := []route.NodeFilterType{route.Healthy, route.MaxHeightForGroup, route.MethodsAllowed, route.NearGlobalMaxHeight} nodeFilter := route.CreateNodeFilter(enabledNodeFilters, healthCheckManager, chainMetadataStore, logger, &chainConfig.Routing) routingStrategy := route.FilteringRoutingStrategy{ NodeFilter: nodeFilter, - BackingStrategy: route.NewPriorityRoundRobinStrategy(logger), + BackingStrategy: route.NewPriorityRoundRobinStrategy(logger, alwaysRoute), Logger: logger, } @@ -72,7 +93,12 @@ func WireDependenciesForAllChains( currentChainConfig := &gatewayConfig.Chains[chainIndex] childLogger := rootLogger.With(zap.String("chainName", currentChainConfig.ChainName)) - dependencyContainer := wireSingleChainDependencies(currentChainConfig, childLogger, rpcCache) + dependencyContainer := wireSingleChainDependencies( + gatewayConfig.Global, + currentChainConfig, + childLogger, + rpcCache, + ) singleChainDependencies = append(singleChainDependencies, dependencyContainer) routers = append(routers, dependencyContainer.router) diff --git a/internal/server/web_server_e2e_test.go b/internal/server/web_server_e2e_test.go index 11d3e6bf..7ee4f695 100644 --- a/internal/server/web_server_e2e_test.go +++ b/internal/server/web_server_e2e_test.go @@ -414,6 +414,9 @@ func handleSingleRequest(t *testing.T, request jsonrpc.SingleRequestBody, case "net_peerCount": return jsonrpc.SingleResponseBody{Result: getResultFromString(hexutil.Uint64(10).String())} + case config.LatencyCheckMethod: + return jsonrpc.SingleResponseBody{Result: getResultFromString(hexutil.Uint64(11).String())} + case "eth_getBlockByNumber": result, _ := json.Marshal(types.Header{ Number: big.NewInt(latestBlockNumber), @@ -451,7 +454,7 @@ func setUpUnhealthyUpstream(t *testing.T) *httptest.Server { switch r := requestBody.(type) { case *jsonrpc.SingleRequestBody: switch requestBody.GetMethod() { - case "eth_syncing", "net_peerCount", "eth_getBlockByNumber": + case "eth_syncing", "net_peerCount", config.LatencyCheckMethod, "eth_getBlockByNumber": responseBody = &jsonrpc.SingleResponseBody{Error: &jsonrpc.Error{Message: "This is a failing fake node!"}} writeResponseBody(t, writer, responseBody) default: diff --git a/internal/types/types.go b/internal/types/types.go index 59790a15..46a3784a 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1,17 +1,28 @@ package types import ( + "time" + "github.com/satsuma-data/node-gateway/internal/config" + "github.com/satsuma-data/node-gateway/internal/jsonrpc" ) type UpstreamStatus struct { BlockHeightCheck BlockHeightChecker PeerCheck Checker SyncingCheck Checker + LatencyCheck LatencyChecker ID string GroupID string } +type RequestData struct { + ResponseBody jsonrpc.ResponseBody + Method string + HTTPResponseCode int + Latency time.Duration +} + //go:generate mockery --output ../mocks --name BlockHeightChecker --with-expecter type BlockHeightChecker interface { RunCheck() @@ -26,4 +37,11 @@ type Checker interface { IsPassing() bool } +//go:generate mockery --output ../mocks --name LatencyChecker --with-expecter +type LatencyChecker interface { + RunPassiveCheck() + GetUnhealthyReason(methods []string) config.UnhealthyReason + RecordRequest(data *RequestData) +} + type PriorityToUpstreamsMap map[int][]*config.UpstreamConfig