From f82d1bcc47910d0884ccd91e3239e6c4d61eac5d Mon Sep 17 00:00:00 2001 From: ziyeqf <51212351+ziyeqf@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:43:31 +0800 Subject: [PATCH] update go-retryablehttp --- go.mod | 4 +- go.sum | 9 +- .../github.com/hashicorp/go-hclog/README.md | 5 +- .../hashicorp/go-hclog/intlogger.go | 135 +++++++++++++++--- .../github.com/hashicorp/go-hclog/logger.go | 22 +++ .../hashicorp/go-retryablehttp/.go-version | 1 + .../hashicorp/go-retryablehttp/CHANGELOG.md | 26 +++- .../hashicorp/go-retryablehttp/CODEOWNERS | 2 +- .../hashicorp/go-retryablehttp/Makefile | 2 +- .../hashicorp/go-retryablehttp/README.md | 2 +- .../go-retryablehttp/cert_error_go119.go | 14 ++ .../go-retryablehttp/cert_error_go120.go | 14 ++ .../hashicorp/go-retryablehttp/client.go | 132 +++++++++++++---- vendor/modules.txt | 6 +- 14 files changed, 302 insertions(+), 72 deletions(-) create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/.go-version create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/cert_error_go119.go create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/cert_error_go120.go diff --git a/go.mod b/go.mod index 42255a79cfcc..e384797c2b43 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/hashicorp/go-azure-helpers v0.69.0 github.com/hashicorp/go-azure-sdk/resource-manager v0.20240507.1095453 github.com/hashicorp/go-azure-sdk/sdk v0.20240507.1095453 - github.com/hashicorp/go-hclog v1.5.0 + github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version v1.6.0 @@ -56,7 +56,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect github.com/hashicorp/go-plugin v1.5.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.5 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/hc-install v0.6.4 // indirect github.com/hashicorp/hcl/v2 v2.18.0 // indirect github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80 // indirect diff --git a/go.sum b/go.sum index edafc31b0858..3fd9575a9b8e 100644 --- a/go.sum +++ b/go.sum @@ -111,16 +111,15 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 h1:1/D3zfFHttUKaCaGKZ/dR2roBXv0vKbSCnssIldfQdI= github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320/go.mod h1:EiZBMaudVLy8fmjf9Npq1dq9RalhveqZG5w/yz3mHWs= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.5.1 h1:oGm7cWBaYIp3lJpx1RUEfLWophprE2EV/KUeqBYo+6k= github.com/hashicorp/go-plugin v1.5.1/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4= -github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= -github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= diff --git a/vendor/github.com/hashicorp/go-hclog/README.md b/vendor/github.com/hashicorp/go-hclog/README.md index 21a17c5af39b..983d44c7db41 100644 --- a/vendor/github.com/hashicorp/go-hclog/README.md +++ b/vendor/github.com/hashicorp/go-hclog/README.md @@ -140,9 +140,10 @@ log.Printf("[DEBUG] %d", 42) ... [DEBUG] my-app: 42 ``` -Notice that if `appLogger` is initialized with the `INFO` log level _and_ you +Notice that if `appLogger` is initialized with the `INFO` log level, _and_ you specify `InferLevels: true`, you will not see any output here. You must change `appLogger` to `DEBUG` to see output. See the docs for more information. If the log lines start with a timestamp you can use the -`InferLevelsWithTimestamp` option to try and ignore them. +`InferLevelsWithTimestamp` option to try and ignore them. Please note that in order +for `InferLevelsWithTimestamp` to be relevant, `InferLevels` must be set to `true`. diff --git a/vendor/github.com/hashicorp/go-hclog/intlogger.go b/vendor/github.com/hashicorp/go-hclog/intlogger.go index b45064acf1a7..272a710c04cc 100644 --- a/vendor/github.com/hashicorp/go-hclog/intlogger.go +++ b/vendor/github.com/hashicorp/go-hclog/intlogger.go @@ -55,23 +55,38 @@ var ( faintBoldColor = color.New(color.Faint, color.Bold) faintColor = color.New(color.Faint) - faintMultiLinePrefix = faintColor.Sprint(" | ") - faintFieldSeparator = faintColor.Sprint("=") - faintFieldSeparatorWithNewLine = faintColor.Sprint("=\n") + faintMultiLinePrefix string + faintFieldSeparator string + faintFieldSeparatorWithNewLine string ) +func init() { + // Force all the colors to enabled because we do our own detection of color usage. + for _, c := range _levelToColor { + c.EnableColor() + } + + faintBoldColor.EnableColor() + faintColor.EnableColor() + + faintMultiLinePrefix = faintColor.Sprint(" | ") + faintFieldSeparator = faintColor.Sprint("=") + faintFieldSeparatorWithNewLine = faintColor.Sprint("=\n") +} + // Make sure that intLogger is a Logger var _ Logger = &intLogger{} // intLogger is an internal logger implementation. Internal in that it is // defined entirely by this package. type intLogger struct { - json bool - callerOffset int - name string - timeFormat string - timeFn TimeFunction - disableTime bool + json bool + jsonEscapeEnabled bool + callerOffset int + name string + timeFormat string + timeFn TimeFunction + disableTime bool // This is an interface so that it's shared by any derived loggers, since // those derived loggers share the bufio.Writer as well. @@ -79,6 +94,19 @@ type intLogger struct { writer *writer level *int32 + // The value of curEpoch when our level was set + setEpoch uint64 + + // The value of curEpoch the last time we performed the level sync process + ownEpoch uint64 + + // Shared amongst all the loggers created in this hierachy, used to determine + // if the level sync process should be run by comparing it with ownEpoch + curEpoch *uint64 + + // The logger this one was created from. Only set when syncParentLevel is set + parent *intLogger + headerColor ColorOption fieldColor ColorOption @@ -88,6 +116,7 @@ type intLogger struct { // create subloggers with their own level setting independentLevels bool + syncParentLevel bool subloggerHook func(sub Logger) Logger } @@ -129,9 +158,9 @@ func newLogger(opts *LoggerOptions) *intLogger { } var ( - primaryColor ColorOption = ColorOff - headerColor ColorOption = ColorOff - fieldColor ColorOption = ColorOff + primaryColor = ColorOff + headerColor = ColorOff + fieldColor = ColorOff ) switch { case opts.ColorHeaderOnly: @@ -145,6 +174,7 @@ func newLogger(opts *LoggerOptions) *intLogger { l := &intLogger{ json: opts.JSONFormat, + jsonEscapeEnabled: !opts.JSONEscapeDisabled, name: opts.Name, timeFormat: TimeFormat, timeFn: time.Now, @@ -152,8 +182,10 @@ func newLogger(opts *LoggerOptions) *intLogger { mutex: mutex, writer: newWriter(output, primaryColor), level: new(int32), + curEpoch: new(uint64), exclude: opts.Exclude, independentLevels: opts.IndependentLevels, + syncParentLevel: opts.SyncParentLevel, headerColor: headerColor, fieldColor: fieldColor, subloggerHook: opts.SubloggerHook, @@ -194,7 +226,7 @@ const offsetIntLogger = 3 // Log a message and a set of key/value pairs if the given level is at // or more severe that the threshold configured in the Logger. func (l *intLogger) log(name string, level Level, msg string, args ...interface{}) { - if level < Level(atomic.LoadInt32(l.level)) { + if level < l.GetLevel() { return } @@ -597,7 +629,7 @@ func (l *intLogger) logJSON(t time.Time, name string, level Level, msg string, a vals := l.jsonMapEntry(t, name, level, msg) args = append(l.implied, args...) - if args != nil && len(args) > 0 { + if len(args) > 0 { if len(args)%2 != 0 { cs, ok := args[len(args)-1].(CapturedStacktrace) if ok { @@ -637,13 +669,17 @@ func (l *intLogger) logJSON(t time.Time, name string, level Level, msg string, a } } - err := json.NewEncoder(l.writer).Encode(vals) + encoder := json.NewEncoder(l.writer) + encoder.SetEscapeHTML(l.jsonEscapeEnabled) + err := encoder.Encode(vals) if err != nil { if _, ok := err.(*json.UnsupportedTypeError); ok { plainVal := l.jsonMapEntry(t, name, level, msg) plainVal["@warn"] = errJsonUnsupportedTypeMsg - json.NewEncoder(l.writer).Encode(plainVal) + errEncoder := json.NewEncoder(l.writer) + errEncoder.SetEscapeHTML(l.jsonEscapeEnabled) + errEncoder.Encode(plainVal) } } } @@ -718,27 +754,27 @@ func (l *intLogger) Error(msg string, args ...interface{}) { // Indicate that the logger would emit TRACE level logs func (l *intLogger) IsTrace() bool { - return Level(atomic.LoadInt32(l.level)) == Trace + return l.GetLevel() == Trace } // Indicate that the logger would emit DEBUG level logs func (l *intLogger) IsDebug() bool { - return Level(atomic.LoadInt32(l.level)) <= Debug + return l.GetLevel() <= Debug } // Indicate that the logger would emit INFO level logs func (l *intLogger) IsInfo() bool { - return Level(atomic.LoadInt32(l.level)) <= Info + return l.GetLevel() <= Info } // Indicate that the logger would emit WARN level logs func (l *intLogger) IsWarn() bool { - return Level(atomic.LoadInt32(l.level)) <= Warn + return l.GetLevel() <= Warn } // Indicate that the logger would emit ERROR level logs func (l *intLogger) IsError() bool { - return Level(atomic.LoadInt32(l.level)) <= Error + return l.GetLevel() <= Error } const MissingKey = "EXTRA_VALUE_AT_END" @@ -854,12 +890,63 @@ func (l *intLogger) resetOutput(opts *LoggerOptions) error { // Update the logging level on-the-fly. This will affect all subloggers as // well. func (l *intLogger) SetLevel(level Level) { - atomic.StoreInt32(l.level, int32(level)) + if !l.syncParentLevel { + atomic.StoreInt32(l.level, int32(level)) + return + } + + nsl := new(int32) + *nsl = int32(level) + + l.level = nsl + + l.ownEpoch = atomic.AddUint64(l.curEpoch, 1) + l.setEpoch = l.ownEpoch +} + +func (l *intLogger) searchLevelPtr() *int32 { + p := l.parent + + ptr := l.level + + max := l.setEpoch + + for p != nil { + if p.setEpoch > max { + max = p.setEpoch + ptr = p.level + } + + p = p.parent + } + + return ptr } // Returns the current level func (l *intLogger) GetLevel() Level { - return Level(atomic.LoadInt32(l.level)) + // We perform the loads immediately to keep the CPU pipeline busy, which + // effectively makes the second load cost nothing. Once loaded into registers + // the comparison returns the already loaded value. The comparison is almost + // always true, so the branch predictor should hit consistently with it. + var ( + curEpoch = atomic.LoadUint64(l.curEpoch) + level = Level(atomic.LoadInt32(l.level)) + own = l.ownEpoch + ) + + if curEpoch == own { + return level + } + + // Perform the level sync process. We'll avoid doing this next time by seeing the + // epoch as current. + + ptr := l.searchLevelPtr() + l.level = ptr + l.ownEpoch = curEpoch + + return Level(atomic.LoadInt32(ptr)) } // Create a *log.Logger that will send it's data through this Logger. This @@ -912,6 +999,8 @@ func (l *intLogger) copy() *intLogger { if l.independentLevels { sl.level = new(int32) *sl.level = *l.level + } else if l.syncParentLevel { + sl.parent = l } return &sl diff --git a/vendor/github.com/hashicorp/go-hclog/logger.go b/vendor/github.com/hashicorp/go-hclog/logger.go index 947ac0c9afc9..ad17544f550e 100644 --- a/vendor/github.com/hashicorp/go-hclog/logger.go +++ b/vendor/github.com/hashicorp/go-hclog/logger.go @@ -233,6 +233,7 @@ type StandardLoggerOptions struct { // [DEBUG] and strip it off before reapplying it. // The timestamp detection may result in false positives and incomplete // string outputs. + // InferLevelsWithTimestamp is only relevant if InferLevels is true. InferLevelsWithTimestamp bool // ForceLevel is used to force all output from the standard logger to be at @@ -263,6 +264,9 @@ type LoggerOptions struct { // Control if the output should be in JSON. JSONFormat bool + // Control the escape switch of json.Encoder + JSONEscapeDisabled bool + // Include file and line information in each log line IncludeLocation bool @@ -303,6 +307,24 @@ type LoggerOptions struct { // will not affect the parent or sibling loggers. IndependentLevels bool + // When set, changing the level of a logger effects only it's direct sub-loggers + // rather than all sub-loggers. For example: + // a := logger.Named("a") + // a.SetLevel(Error) + // b := a.Named("b") + // c := a.Named("c") + // b.GetLevel() => Error + // c.GetLevel() => Error + // b.SetLevel(Info) + // a.GetLevel() => Error + // b.GetLevel() => Info + // c.GetLevel() => Error + // a.SetLevel(Warn) + // a.GetLevel() => Warn + // b.GetLevel() => Warn + // c.GetLevel() => Warn + SyncParentLevel bool + // SubloggerHook registers a function that is called when a sublogger via // Named, With, or ResetNamed is created. If defined, the function is passed // the newly created Logger and the returned Logger is returned from the diff --git a/vendor/github.com/hashicorp/go-retryablehttp/.go-version b/vendor/github.com/hashicorp/go-retryablehttp/.go-version new file mode 100644 index 000000000000..6fee2fedb0a4 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/.go-version @@ -0,0 +1 @@ +1.22.2 diff --git a/vendor/github.com/hashicorp/go-retryablehttp/CHANGELOG.md b/vendor/github.com/hashicorp/go-retryablehttp/CHANGELOG.md index 7a17b9f99309..68a627c6d97c 100644 --- a/vendor/github.com/hashicorp/go-retryablehttp/CHANGELOG.md +++ b/vendor/github.com/hashicorp/go-retryablehttp/CHANGELOG.md @@ -1,14 +1,32 @@ +## 0.7.7 (May 30, 2024) + +BUG FIXES: + +- client: avoid potentially leaking URL-embedded basic authentication credentials in logs (#158) + +## 0.7.6 (May 9, 2024) + +ENHANCEMENTS: + +- client: support a `RetryPrepare` function for modifying the request before retrying (#216) +- client: support HTTP-date values for `Retry-After` header value (#138) +- client: avoid reading entire body when the body is a `*bytes.Reader` (#197) + +BUG FIXES: + +- client: fix a broken check for invalid server certificate in go 1.20+ (#210) + ## 0.7.5 (Nov 8, 2023) -BUG FIXES +BUG FIXES: -- client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections [GH-207] +- client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections (#207) ## 0.7.4 (Jun 6, 2023) -BUG FIXES +BUG FIXES: -- client: fixing an issue where the Content-Type header wouldn't be sent with an empty payload when using HTTP/2 [GH-194] +- client: fixing an issue where the Content-Type header wouldn't be sent with an empty payload when using HTTP/2 (#194) ## 0.7.3 (May 15, 2023) diff --git a/vendor/github.com/hashicorp/go-retryablehttp/CODEOWNERS b/vendor/github.com/hashicorp/go-retryablehttp/CODEOWNERS index f8389c995e66..d6dd78a2dd9c 100644 --- a/vendor/github.com/hashicorp/go-retryablehttp/CODEOWNERS +++ b/vendor/github.com/hashicorp/go-retryablehttp/CODEOWNERS @@ -1 +1 @@ -* @hashicorp/release-engineering \ No newline at end of file +* @hashicorp/go-retryablehttp-maintainers diff --git a/vendor/github.com/hashicorp/go-retryablehttp/Makefile b/vendor/github.com/hashicorp/go-retryablehttp/Makefile index da17640e644c..5255241961eb 100644 --- a/vendor/github.com/hashicorp/go-retryablehttp/Makefile +++ b/vendor/github.com/hashicorp/go-retryablehttp/Makefile @@ -2,7 +2,7 @@ default: test test: go vet ./... - go test -race ./... + go test -v -race ./... updatedeps: go get -f -t -u ./... diff --git a/vendor/github.com/hashicorp/go-retryablehttp/README.md b/vendor/github.com/hashicorp/go-retryablehttp/README.md index 8943becf19bf..145a62f218cb 100644 --- a/vendor/github.com/hashicorp/go-retryablehttp/README.md +++ b/vendor/github.com/hashicorp/go-retryablehttp/README.md @@ -59,4 +59,4 @@ standardClient := retryClient.StandardClient() // *http.Client ``` For more usage and examples see the -[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp). +[pkg.go.dev](https://pkg.go.dev/github.com/hashicorp/go-retryablehttp). diff --git a/vendor/github.com/hashicorp/go-retryablehttp/cert_error_go119.go b/vendor/github.com/hashicorp/go-retryablehttp/cert_error_go119.go new file mode 100644 index 000000000000..b2b27e87225c --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/cert_error_go119.go @@ -0,0 +1,14 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !go1.20 +// +build !go1.20 + +package retryablehttp + +import "crypto/x509" + +func isCertError(err error) bool { + _, ok := err.(x509.UnknownAuthorityError) + return ok +} diff --git a/vendor/github.com/hashicorp/go-retryablehttp/cert_error_go120.go b/vendor/github.com/hashicorp/go-retryablehttp/cert_error_go120.go new file mode 100644 index 000000000000..a3cd315a28b7 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/cert_error_go120.go @@ -0,0 +1,14 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build go1.20 +// +build go1.20 + +package retryablehttp + +import "crypto/tls" + +func isCertError(err error) bool { + _, ok := err.(*tls.CertificateVerificationError) + return ok +} diff --git a/vendor/github.com/hashicorp/go-retryablehttp/client.go b/vendor/github.com/hashicorp/go-retryablehttp/client.go index c9edbd0595b0..efee53c400d2 100644 --- a/vendor/github.com/hashicorp/go-retryablehttp/client.go +++ b/vendor/github.com/hashicorp/go-retryablehttp/client.go @@ -27,10 +27,8 @@ package retryablehttp import ( "bytes" "context" - "crypto/x509" "fmt" "io" - "io/ioutil" "log" "math" "math/rand" @@ -63,6 +61,10 @@ var ( // limit the size we consume to respReadLimit. respReadLimit = int64(4096) + // timeNow sets the function that returns the current time. + // This defaults to time.Now. Changes to this should only be done in tests. + timeNow = time.Now + // A regular expression to match the error returned by net/http when the // configured number of redirects is exhausted. This error isn't typed // specifically so we resort to matching on the error string. @@ -73,6 +75,11 @@ var ( // specifically so we resort to matching on the error string. schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`) + // A regular expression to match the error returned by net/http when a + // request header or value is invalid. This error isn't typed + // specifically so we resort to matching on the error string. + invalidHeaderErrorRe = regexp.MustCompile(`invalid header`) + // A regular expression to match the error returned by net/http when the // TLS certificate is not trusted. This error isn't typed // specifically so we resort to matching on the error string. @@ -248,21 +255,19 @@ func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, erro // deal with it seeking so want it to match here instead of the // io.ReadSeeker case. case *bytes.Reader: - buf, err := ioutil.ReadAll(body) - if err != nil { - return nil, 0, err - } + snapshot := *body bodyReader = func() (io.Reader, error) { - return bytes.NewReader(buf), nil + r := snapshot + return &r, nil } - contentLength = int64(len(buf)) + contentLength = int64(body.Len()) // Compat case case io.ReadSeeker: raw := body bodyReader = func() (io.Reader, error) { _, err := raw.Seek(0, 0) - return ioutil.NopCloser(raw), err + return io.NopCloser(raw), err } if lr, ok := raw.(LenReader); ok { contentLength = int64(lr.Len()) @@ -270,7 +275,7 @@ func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, erro // Read all in so we can reset case io.Reader: - buf, err := ioutil.ReadAll(body) + buf, err := io.ReadAll(body) if err != nil { return nil, 0, err } @@ -393,6 +398,9 @@ type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) t // attempted. If overriding this, be sure to close the body if needed. type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error) +// PrepareRetry is called before retry operation. It can be used for example to re-sign the request +type PrepareRetry func(req *http.Request) error + // Client is used to make HTTP requests. It adds additional functionality // like automatic retries to tolerate minor outages. type Client struct { @@ -421,6 +429,9 @@ type Client struct { // ErrorHandler specifies the custom error handler to use, if any ErrorHandler ErrorHandler + // PrepareRetry can prepare the request for retry operation, for example re-sign it + PrepareRetry PrepareRetry + loggerInit sync.Once clientInit sync.Once } @@ -494,11 +505,16 @@ func baseRetryPolicy(resp *http.Response, err error) (bool, error) { return false, v } + // Don't retry if the error was due to an invalid header. + if invalidHeaderErrorRe.MatchString(v.Error()) { + return false, v + } + // Don't retry if the error was due to TLS cert verification failure. if notTrustedErrorRe.MatchString(v.Error()) { return false, v } - if _, ok := v.Err.(x509.UnknownAuthorityError); ok { + if isCertError(v.Err) { return false, v } } @@ -535,10 +551,8 @@ func baseRetryPolicy(resp *http.Response, err error) (bool, error) { func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { if resp != nil { if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable { - if s, ok := resp.Header["Retry-After"]; ok { - if sleep, err := strconv.ParseInt(s[0], 10, 64); err == nil { - return time.Second * time.Duration(sleep) - } + if sleep, ok := parseRetryAfterHeader(resp.Header["Retry-After"]); ok { + return sleep } } } @@ -551,6 +565,41 @@ func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) return sleep } +// parseRetryAfterHeader parses the Retry-After header and returns the +// delay duration according to the spec: https://httpwg.org/specs/rfc7231.html#header.retry-after +// The bool returned will be true if the header was successfully parsed. +// Otherwise, the header was either not present, or was not parseable according to the spec. +// +// Retry-After headers come in two flavors: Seconds or HTTP-Date +// +// Examples: +// * Retry-After: Fri, 31 Dec 1999 23:59:59 GMT +// * Retry-After: 120 +func parseRetryAfterHeader(headers []string) (time.Duration, bool) { + if len(headers) == 0 || headers[0] == "" { + return 0, false + } + header := headers[0] + // Retry-After: 120 + if sleep, err := strconv.ParseInt(header, 10, 64); err == nil { + if sleep < 0 { // a negative sleep doesn't make sense + return 0, false + } + return time.Second * time.Duration(sleep), true + } + + // Retry-After: Fri, 31 Dec 1999 23:59:59 GMT + retryTime, err := time.Parse(time.RFC1123, header) + if err != nil { + return 0, false + } + if until := retryTime.Sub(timeNow()); until > 0 { + return until, true + } + // date is in the past + return 0, true +} + // LinearJitterBackoff provides a callback for Client.Backoff which will // perform linear backoff based on the attempt number and with jitter to // prevent a thundering herd. @@ -578,13 +627,13 @@ func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Resp } // Seed rand; doing this every time is fine - rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) + source := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) // Pick a random number that lies somewhere between the min and max and // multiply by the attemptNum. attemptNum starts at zero so we always // increment here. We first get a random percentage, then apply that to the // difference between min and max, and add to min. - jitter := rand.Float64() * float64(max-min) + jitter := source.Float64() * float64(max-min) jitterMin := int64(jitter) + int64(min) return time.Duration(jitterMin * int64(attemptNum)) } @@ -609,19 +658,19 @@ func (c *Client) Do(req *Request) (*http.Response, error) { if logger != nil { switch v := logger.(type) { case LeveledLogger: - v.Debug("performing request", "method", req.Method, "url", req.URL) + v.Debug("performing request", "method", req.Method, "url", redactURL(req.URL)) case Logger: - v.Printf("[DEBUG] %s %s", req.Method, req.URL) + v.Printf("[DEBUG] %s %s", req.Method, redactURL(req.URL)) } } var resp *http.Response var attempt int var shouldRetry bool - var doErr, respErr, checkErr error + var doErr, respErr, checkErr, prepareErr error for i := 0; ; i++ { - doErr, respErr = nil, nil + doErr, respErr, prepareErr = nil, nil, nil attempt++ // Always rewind the request body when non-nil. @@ -634,7 +683,7 @@ func (c *Client) Do(req *Request) (*http.Response, error) { if c, ok := body.(io.ReadCloser); ok { req.Body = c } else { - req.Body = ioutil.NopCloser(body) + req.Body = io.NopCloser(body) } } @@ -666,9 +715,9 @@ func (c *Client) Do(req *Request) (*http.Response, error) { if err != nil { switch v := logger.(type) { case LeveledLogger: - v.Error("request failed", "error", err, "method", req.Method, "url", req.URL) + v.Error("request failed", "error", err, "method", req.Method, "url", redactURL(req.URL)) case Logger: - v.Printf("[ERR] %s %s request failed: %v", req.Method, req.URL, err) + v.Printf("[ERR] %s %s request failed: %v", req.Method, redactURL(req.URL), err) } } else { // Call this here to maintain the behavior of logging all requests, @@ -704,7 +753,7 @@ func (c *Client) Do(req *Request) (*http.Response, error) { wait := c.Backoff(c.RetryWaitMin, c.RetryWaitMax, i, resp) if logger != nil { - desc := fmt.Sprintf("%s %s", req.Method, req.URL) + desc := fmt.Sprintf("%s %s", req.Method, redactURL(req.URL)) if resp != nil { desc = fmt.Sprintf("%s (status: %d)", desc, resp.StatusCode) } @@ -728,17 +777,26 @@ func (c *Client) Do(req *Request) (*http.Response, error) { // without racing against the closeBody call in persistConn.writeLoop. httpreq := *req.Request req.Request = &httpreq + + if c.PrepareRetry != nil { + if err := c.PrepareRetry(req.Request); err != nil { + prepareErr = err + break + } + } } // this is the closest we have to success criteria - if doErr == nil && respErr == nil && checkErr == nil && !shouldRetry { + if doErr == nil && respErr == nil && checkErr == nil && prepareErr == nil && !shouldRetry { return resp, nil } defer c.HTTPClient.CloseIdleConnections() var err error - if checkErr != nil { + if prepareErr != nil { + err = prepareErr + } else if checkErr != nil { err = checkErr } else if respErr != nil { err = respErr @@ -760,17 +818,17 @@ func (c *Client) Do(req *Request) (*http.Response, error) { // communicate why if err == nil { return nil, fmt.Errorf("%s %s giving up after %d attempt(s)", - req.Method, req.URL, attempt) + req.Method, redactURL(req.URL), attempt) } return nil, fmt.Errorf("%s %s giving up after %d attempt(s): %w", - req.Method, req.URL, attempt, err) + req.Method, redactURL(req.URL), attempt, err) } // Try to read the response body so we can reuse this connection. func (c *Client) drainBody(body io.ReadCloser) { defer body.Close() - _, err := io.Copy(ioutil.Discard, io.LimitReader(body, respReadLimit)) + _, err := io.Copy(io.Discard, io.LimitReader(body, respReadLimit)) if err != nil { if c.logger() != nil { switch v := c.logger().(type) { @@ -845,3 +903,17 @@ func (c *Client) StandardClient() *http.Client { Transport: &RoundTripper{Client: c}, } } + +// Taken from url.URL#Redacted() which was introduced in go 1.15. +// We can switch to using it directly if we'll bump the minimum required go version. +func redactURL(u *url.URL) string { + if u == nil { + return "" + } + + ru := *u + if _, has := ru.User.Password(); has { + ru.User = url.UserPassword(ru.User.Username(), "xxxxx") + } + return ru.String() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 094d90822797..8bdc7cca4546 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1185,7 +1185,7 @@ github.com/hashicorp/go-cty/cty/gocty github.com/hashicorp/go-cty/cty/json github.com/hashicorp/go-cty/cty/msgpack github.com/hashicorp/go-cty/cty/set -# github.com/hashicorp/go-hclog v1.5.0 +# github.com/hashicorp/go-hclog v1.6.3 ## explicit; go 1.13 github.com/hashicorp/go-hclog # github.com/hashicorp/go-multierror v1.1.1 @@ -1197,8 +1197,8 @@ github.com/hashicorp/go-plugin github.com/hashicorp/go-plugin/internal/cmdrunner github.com/hashicorp/go-plugin/internal/plugin github.com/hashicorp/go-plugin/runner -# github.com/hashicorp/go-retryablehttp v0.7.5 -## explicit; go 1.13 +# github.com/hashicorp/go-retryablehttp v0.7.7 +## explicit; go 1.19 github.com/hashicorp/go-retryablehttp # github.com/hashicorp/go-uuid v1.0.3 ## explicit