From d81937a13f93fc9863af1fb1466753509e78c385 Mon Sep 17 00:00:00 2001
From: duyiwei <duyiwei@kylinos.cn>
Date: Tue, 14 Jan 2025 14:22:39 +0800
Subject: [PATCH] fix CVE-2024-6104
Signed-off-by: duyiwei <duyiwei@kylinos.cn>
go.mod | 2 +-
go.sum | 4 +-
.../hashicorp/go-retryablehttp/CHANGELOG.md | 28 +++-
.../hashicorp/go-retryablehttp/CODEOWNERS | 2 +-
.../hashicorp/go-retryablehttp/Makefile | 2 +-
.../hashicorp/go-retryablehttp/README.md | 2 +-
.../go-retryablehttp/cert_error_go119.go | 14 ++
.../go-retryablehttp/cert_error_go120.go | 14 ++
.../hashicorp/go-retryablehttp/client.go | 132 ++++++++++++++----
vendor/modules.txt | 4 +-
10 files changed, 161 insertions(+), 43 deletions(-)
create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/cert_error_go119.go
create mode 100644 vendor/github.com/hashicorp/go-retryablehttp/cert_error_go120.go
@@ -142,7 +142,7 @@ require (
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
- github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
+ github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
@@ -646,8 +646,8 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
-github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -1,15 +1,33 @@
+## 0.7.7 (May 30, 2024)
+
+BUG FIXES:
+
+- client: avoid potentially leaking URL-embedded basic authentication credentials in logs (#158)
+
+## 0.7.6 (May 9, 2024)
+
+ENHANCEMENTS:
+
+- client: support a `RetryPrepare` function for modifying the request before retrying (#216)
+- client: support HTTP-date values for `Retry-After` header value (#138)
+- client: avoid reading entire body when the body is a `*bytes.Reader` (#197)
+
+BUG FIXES:
+
+- client: fix a broken check for invalid server certificate in go 1.20+ (#210)
+
## 0.7.5 (Nov 8, 2023)
-BUG FIXES
+BUG FIXES:
-- client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections [GH-207]
+- client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections (#207)
## 0.7.4 (Jun 6, 2023)
-BUG FIXES
+BUG FIXES:
-- client: fixing an issue where the Content-Type header wouldn't be sent with an empty payload when using HTTP/2 [GH-194]
+- client: fixing an issue where the Content-Type header wouldn't be sent with an empty payload when using HTTP/2 (#194)
## 0.7.3 (May 15, 2023)
-Initial release
+Initial release
\ No newline at end of file
@@ -1 +1 @@
-* @hashicorp/release-engineering
\ No newline at end of file
+* @hashicorp/go-retryablehttp-maintainers
\ No newline at end of file
@@ -2,7 +2,7 @@ default: test
test:
go vet ./...
- go test -race ./...
+ go test -v -race ./...
updatedeps:
go get -f -t -u ./...
@@ -59,4 +59,4 @@ standardClient := retryClient.StandardClient() // *http.Client
```
For more usage and examples see the
-[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp).
+[pkg.go.dev](https://pkg.go.dev/github.com/hashicorp/go-retryablehttp).
new file mode 100644
@@ -0,0 +1,14 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+//go:build !go1.20
+// +build !go1.20
+
+package retryablehttp
+
+import "crypto/x509"
+
+func isCertError(err error) bool {
+ _, ok := err.(x509.UnknownAuthorityError)
+ return ok
+}
new file mode 100644
@@ -0,0 +1,14 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+//go:build go1.20
+// +build go1.20
+
+package retryablehttp
+
+import "crypto/tls"
+
+func isCertError(err error) bool {
+ _, ok := err.(*tls.CertificateVerificationError)
+ return ok
+}
@@ -27,10 +27,8 @@ package retryablehttp
import (
"bytes"
"context"
- "crypto/x509"
"fmt"
"io"
- "io/ioutil"
"log"
"math"
"math/rand"
@@ -63,6 +61,10 @@ var (
// limit the size we consume to respReadLimit.
respReadLimit = int64(4096)
+ // timeNow sets the function that returns the current time.
+ // This defaults to time.Now. Changes to this should only be done in tests.
+ timeNow = time.Now
+
// A regular expression to match the error returned by net/http when the
// configured number of redirects is exhausted. This error isn't typed
// specifically so we resort to matching on the error string.
@@ -73,6 +75,11 @@ var (
// specifically so we resort to matching on the error string.
schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`)
+ // A regular expression to match the error returned by net/http when a
+ // request header or value is invalid. This error isn't typed
+ // specifically so we resort to matching on the error string.
+ invalidHeaderErrorRe = regexp.MustCompile(`invalid header`)
+
// A regular expression to match the error returned by net/http when the
// TLS certificate is not trusted. This error isn't typed
// specifically so we resort to matching on the error string.
@@ -248,21 +255,19 @@ func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, erro
// deal with it seeking so want it to match here instead of the
// io.ReadSeeker case.
case *bytes.Reader:
- buf, err := ioutil.ReadAll(body)
- if err != nil {
- return nil, 0, err
- }
+ snapshot := *body
bodyReader = func() (io.Reader, error) {
- return bytes.NewReader(buf), nil
+ r := snapshot
+ return &r, nil
}
- contentLength = int64(len(buf))
+ contentLength = int64(body.Len())
// Compat case
case io.ReadSeeker:
raw := body
bodyReader = func() (io.Reader, error) {
_, err := raw.Seek(0, 0)
- return ioutil.NopCloser(raw), err
+ return io.NopCloser(raw), err
}
if lr, ok := raw.(LenReader); ok {
contentLength = int64(lr.Len())
@@ -270,7 +275,7 @@ func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, erro
// Read all in so we can reset
case io.Reader:
- buf, err := ioutil.ReadAll(body)
+ buf, err := io.ReadAll(body)
if err != nil {
return nil, 0, err
}
@@ -393,6 +398,9 @@ type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) t
// attempted. If overriding this, be sure to close the body if needed.
type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error)
+// PrepareRetry is called before retry operation. It can be used for example to re-sign the request
+type PrepareRetry func(req *http.Request) error
+
// Client is used to make HTTP requests. It adds additional functionality
// like automatic retries to tolerate minor outages.
type Client struct {
@@ -421,6 +429,9 @@ type Client struct {
// ErrorHandler specifies the custom error handler to use, if any
ErrorHandler ErrorHandler
+ // PrepareRetry can prepare the request for retry operation, for example re-sign it
+ PrepareRetry PrepareRetry
+
loggerInit sync.Once
clientInit sync.Once
}
@@ -494,11 +505,16 @@ func baseRetryPolicy(resp *http.Response, err error) (bool, error) {
return false, v
}
+ // Don't retry if the error was due to an invalid header.
+ if invalidHeaderErrorRe.MatchString(v.Error()) {
+ return false, v
+ }
+
// Don't retry if the error was due to TLS cert verification failure.
if notTrustedErrorRe.MatchString(v.Error()) {
return false, v
}
- if _, ok := v.Err.(x509.UnknownAuthorityError); ok {
+ if isCertError(v.Err) {
return false, v
}
}
@@ -535,10 +551,8 @@ func baseRetryPolicy(resp *http.Response, err error) (bool, error) {
func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
if resp != nil {
if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
- if s, ok := resp.Header["Retry-After"]; ok {
- if sleep, err := strconv.ParseInt(s[0], 10, 64); err == nil {
- return time.Second * time.Duration(sleep)
- }
+ if sleep, ok := parseRetryAfterHeader(resp.Header["Retry-After"]); ok {
+ return sleep
}
}
}
@@ -551,6 +565,41 @@ func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response)
return sleep
}
+// parseRetryAfterHeader parses the Retry-After header and returns the
+// delay duration according to the spec: https://httpwg.org/specs/rfc7231.html#header.retry-after
+// The bool returned will be true if the header was successfully parsed.
+// Otherwise, the header was either not present, or was not parseable according to the spec.
+//
+// Retry-After headers come in two flavors: Seconds or HTTP-Date
+//
+// Examples:
+// * Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
+// * Retry-After: 120
+func parseRetryAfterHeader(headers []string) (time.Duration, bool) {
+ if len(headers) == 0 || headers[0] == "" {
+ return 0, false
+ }
+ header := headers[0]
+ // Retry-After: 120
+ if sleep, err := strconv.ParseInt(header, 10, 64); err == nil {
+ if sleep < 0 { // a negative sleep doesn't make sense
+ return 0, false
+ }
+ return time.Second * time.Duration(sleep), true
+ }
+
+ // Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
+ retryTime, err := time.Parse(time.RFC1123, header)
+ if err != nil {
+ return 0, false
+ }
+ if until := retryTime.Sub(timeNow()); until > 0 {
+ return until, true
+ }
+ // date is in the past
+ return 0, true
+}
+
// LinearJitterBackoff provides a callback for Client.Backoff which will
// perform linear backoff based on the attempt number and with jitter to
// prevent a thundering herd.
@@ -578,13 +627,13 @@ func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Resp
}
// Seed rand; doing this every time is fine
- rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
+ source := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
// Pick a random number that lies somewhere between the min and max and
// multiply by the attemptNum. attemptNum starts at zero so we always
// increment here. We first get a random percentage, then apply that to the
// difference between min and max, and add to min.
- jitter := rand.Float64() * float64(max-min)
+ jitter := source.Float64() * float64(max-min)
jitterMin := int64(jitter) + int64(min)
return time.Duration(jitterMin * int64(attemptNum))
}
@@ -609,19 +658,19 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
if logger != nil {
switch v := logger.(type) {
case LeveledLogger:
- v.Debug("performing request", "method", req.Method, "url", req.URL)
+ v.Debug("performing request", "method", req.Method, "url", redactURL(req.URL))
case Logger:
- v.Printf("[DEBUG] %s %s", req.Method, req.URL)
+ v.Printf("[DEBUG] %s %s", req.Method, redactURL(req.URL))
}
}
var resp *http.Response
var attempt int
var shouldRetry bool
- var doErr, respErr, checkErr error
+ var doErr, respErr, checkErr, prepareErr error
for i := 0; ; i++ {
- doErr, respErr = nil, nil
+ doErr, respErr, prepareErr = nil, nil, nil
attempt++
// Always rewind the request body when non-nil.
@@ -634,7 +683,7 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
if c, ok := body.(io.ReadCloser); ok {
req.Body = c
} else {
- req.Body = ioutil.NopCloser(body)
+ req.Body = io.NopCloser(body)
}
}
@@ -666,9 +715,9 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
if err != nil {
switch v := logger.(type) {
case LeveledLogger:
- v.Error("request failed", "error", err, "method", req.Method, "url", req.URL)
+ v.Error("request failed", "error", err, "method", req.Method, "url", redactURL(req.URL))
case Logger:
- v.Printf("[ERR] %s %s request failed: %v", req.Method, req.URL, err)
+ v.Printf("[ERR] %s %s request failed: %v", req.Method, redactURL(req.URL), err)
}
} else {
// Call this here to maintain the behavior of logging all requests,
@@ -704,7 +753,7 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
wait := c.Backoff(c.RetryWaitMin, c.RetryWaitMax, i, resp)
if logger != nil {
- desc := fmt.Sprintf("%s %s", req.Method, req.URL)
+ desc := fmt.Sprintf("%s %s", req.Method, redactURL(req.URL))
if resp != nil {
desc = fmt.Sprintf("%s (status: %d)", desc, resp.StatusCode)
}
@@ -728,17 +777,26 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
// without racing against the closeBody call in persistConn.writeLoop.
httpreq := *req.Request
req.Request = &httpreq
+
+ if c.PrepareRetry != nil {
+ if err := c.PrepareRetry(req.Request); err != nil {
+ prepareErr = err
+ break
+ }
+ }
}
// this is the closest we have to success criteria
- if doErr == nil && respErr == nil && checkErr == nil && !shouldRetry {
+ if doErr == nil && respErr == nil && checkErr == nil && prepareErr == nil && !shouldRetry {
return resp, nil
}
defer c.HTTPClient.CloseIdleConnections()
var err error
- if checkErr != nil {
+ if prepareErr != nil {
+ err = prepareErr
+ } else if checkErr != nil {
err = checkErr
} else if respErr != nil {
err = respErr
@@ -760,17 +818,17 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
// communicate why
if err == nil {
return nil, fmt.Errorf("%s %s giving up after %d attempt(s)",
- req.Method, req.URL, attempt)
+ req.Method, redactURL(req.URL), attempt)
}
return nil, fmt.Errorf("%s %s giving up after %d attempt(s): %w",
- req.Method, req.URL, attempt, err)
+ req.Method, redactURL(req.URL), attempt, err)
}
// Try to read the response body so we can reuse this connection.
func (c *Client) drainBody(body io.ReadCloser) {
defer body.Close()
- _, err := io.Copy(ioutil.Discard, io.LimitReader(body, respReadLimit))
+ _, err := io.Copy(io.Discard, io.LimitReader(body, respReadLimit))
if err != nil {
if c.logger() != nil {
switch v := c.logger().(type) {
@@ -845,3 +903,17 @@ func (c *Client) StandardClient() *http.Client {
Transport: &RoundTripper{Client: c},
}
}
+
+// Taken from url.URL#Redacted() which was introduced in go 1.15.
+// We can switch to using it directly if we'll bump the minimum required go version.
+func redactURL(u *url.URL) string {
+ if u == nil {
+ return ""
+ }
+
+ ru := *u
+ if _, has := ru.User.Password(); has {
+ ru.User = url.UserPassword(ru.User.Username(), "xxxxx")
+ }
+ return ru.String()
+}
@@ -685,8 +685,8 @@ github.com/hashicorp/go-cleanhttp
# github.com/hashicorp/go-multierror v1.1.1
## explicit; go 1.13
github.com/hashicorp/go-multierror
-# github.com/hashicorp/go-retryablehttp v0.7.5
-## explicit; go 1.13
+# github.com/hashicorp/go-retryablehttp v0.7.7
+## explicit; go 1.19
github.com/hashicorp/go-retryablehttp
# github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921
## explicit; go 1.20
--
2.33.0