Skip to content
Snippets Groups Projects
Verified Commit eb53d644 authored by Volker Schukai's avatar Volker Schukai :alien:
Browse files

chore: update go vendor

parent 4e2ab573
No related branches found
No related tags found
No related merge requests found
Showing
with 2170 additions and 0 deletions
package ratelimit
import (
"context"
"fmt"
)
type rateToken struct {
tokenCost uint
bucket *TokenBucket
}
func (t rateToken) release() error {
t.bucket.Refund(t.tokenCost)
return nil
}
// TokenRateLimit provides a Token Bucket RateLimiter implementation
// that limits the overall number of retry attempts that can be made across
// operation invocations.
type TokenRateLimit struct {
bucket *TokenBucket
}
// NewTokenRateLimit returns an TokenRateLimit with default values.
// Functional options can configure the retry rate limiter.
func NewTokenRateLimit(tokens uint) *TokenRateLimit {
return &TokenRateLimit{
bucket: NewTokenBucket(tokens),
}
}
type canceledError struct {
Err error
}
func (c canceledError) CanceledError() bool { return true }
func (c canceledError) Unwrap() error { return c.Err }
func (c canceledError) Error() string {
return fmt.Sprintf("canceled, %v", c.Err)
}
// GetToken may cause a available pool of retry quota to be
// decremented. Will return an error if the decremented value can not be
// reduced from the retry quota.
func (l *TokenRateLimit) GetToken(ctx context.Context, cost uint) (func() error, error) {
select {
case <-ctx.Done():
return nil, canceledError{Err: ctx.Err()}
default:
}
if avail, ok := l.bucket.Retrieve(cost); !ok {
return nil, QuotaExceededError{Available: avail, Requested: cost}
}
return rateToken{
tokenCost: cost,
bucket: l.bucket,
}.release, nil
}
// AddTokens increments the token bucket by a fixed amount.
func (l *TokenRateLimit) AddTokens(v uint) error {
l.bucket.Refund(v)
return nil
}
// Remaining returns the number of remaining tokens in the bucket.
func (l *TokenRateLimit) Remaining() uint {
return l.bucket.Remaining()
}
// QuotaExceededError provides the SDK error when the retries for a given
// token bucket have been exhausted.
type QuotaExceededError struct {
Available uint
Requested uint
}
func (e QuotaExceededError) Error() string {
return fmt.Sprintf("retry quota exceeded, %d available, %d requested",
e.Available, e.Requested)
}
package aws
import (
"fmt"
)
// TODO remove replace with smithy.CanceledError
// RequestCanceledError is the error that will be returned by an API request
// that was canceled. Requests given a Context may return this error when
// canceled.
type RequestCanceledError struct {
Err error
}
// CanceledError returns true to satisfy interfaces checking for canceled errors.
func (*RequestCanceledError) CanceledError() bool { return true }
// Unwrap returns the underlying error, if there was one.
func (e *RequestCanceledError) Unwrap() error {
return e.Err
}
func (e *RequestCanceledError) Error() string {
return fmt.Sprintf("request canceled, %v", e.Err)
}
package retry
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
)
const (
// DefaultRequestCost is the cost of a single request from the adaptive
// rate limited token bucket.
DefaultRequestCost uint = 1
)
// DefaultThrottles provides the set of errors considered throttle errors that
// are checked by default.
var DefaultThrottles = []IsErrorThrottle{
ThrottleErrorCode{
Codes: DefaultThrottleErrorCodes,
},
}
// AdaptiveModeOptions provides the functional options for configuring the
// adaptive retry mode, and delay behavior.
type AdaptiveModeOptions struct {
// If the adaptive token bucket is empty, when an attempt will be made
// AdaptiveMode will sleep until a token is available. This can occur when
// attempts fail with throttle errors. Use this option to disable the sleep
// until token is available, and return error immediately.
FailOnNoAttemptTokens bool
// The cost of an attempt from the AdaptiveMode's adaptive token bucket.
RequestCost uint
// Set of strategies to determine if the attempt failed due to a throttle
// error.
//
// It is safe to append to this list in NewAdaptiveMode's functional options.
Throttles []IsErrorThrottle
// Set of options for standard retry mode that AdaptiveMode is built on top
// of. AdaptiveMode may apply its own defaults to Standard retry mode that
// are different than the defaults of NewStandard. Use these options to
// override the default options.
StandardOptions []func(*StandardOptions)
}
// AdaptiveMode provides an experimental retry strategy that expands on the
// Standard retry strategy, adding client attempt rate limits. The attempt rate
// limit is initially unrestricted, but becomes restricted when the attempt
// fails with for a throttle error. When restricted AdaptiveMode may need to
// sleep before an attempt is made, if too many throttles have been received.
// AdaptiveMode's sleep can be canceled with context cancel. Set
// AdaptiveModeOptions FailOnNoAttemptTokens to change the behavior from sleep,
// to fail fast.
//
// Eventually unrestricted attempt rate limit will be restored once attempts no
// longer are failing due to throttle errors.
type AdaptiveMode struct {
options AdaptiveModeOptions
throttles IsErrorThrottles
retryer aws.RetryerV2
rateLimit *adaptiveRateLimit
}
// NewAdaptiveMode returns an initialized AdaptiveMode retry strategy.
func NewAdaptiveMode(optFns ...func(*AdaptiveModeOptions)) *AdaptiveMode {
o := AdaptiveModeOptions{
RequestCost: DefaultRequestCost,
Throttles: append([]IsErrorThrottle{}, DefaultThrottles...),
}
for _, fn := range optFns {
fn(&o)
}
return &AdaptiveMode{
options: o,
throttles: IsErrorThrottles(o.Throttles),
retryer: NewStandard(o.StandardOptions...),
rateLimit: newAdaptiveRateLimit(),
}
}
// IsErrorRetryable returns if the failed attempt is retryable. This check
// should determine if the error can be retried, or if the error is
// terminal.
func (a *AdaptiveMode) IsErrorRetryable(err error) bool {
return a.retryer.IsErrorRetryable(err)
}
// MaxAttempts returns the maximum number of attempts that can be made for
// an attempt before failing. A value of 0 implies that the attempt should
// be retried until it succeeds if the errors are retryable.
func (a *AdaptiveMode) MaxAttempts() int {
return a.retryer.MaxAttempts()
}
// RetryDelay returns the delay that should be used before retrying the
// attempt. Will return error if the if the delay could not be determined.
func (a *AdaptiveMode) RetryDelay(attempt int, opErr error) (
time.Duration, error,
) {
return a.retryer.RetryDelay(attempt, opErr)
}
// GetRetryToken attempts to deduct the retry cost from the retry token pool.
// Returning the token release function, or error.
func (a *AdaptiveMode) GetRetryToken(ctx context.Context, opErr error) (
releaseToken func(error) error, err error,
) {
return a.retryer.GetRetryToken(ctx, opErr)
}
// GetInitialToken returns the initial attempt token that can increment the
// retry token pool if the attempt is successful.
//
// Deprecated: This method does not provide a way to block using Context,
// nor can it return an error. Use RetryerV2, and GetAttemptToken instead. Only
// present to implement Retryer interface.
func (a *AdaptiveMode) GetInitialToken() (releaseToken func(error) error) {
return nopRelease
}
// GetAttemptToken returns the attempt token that can be used to rate limit
// attempt calls. Will be used by the SDK's retry package's Attempt
// middleware to get an attempt token prior to calling the temp and releasing
// the attempt token after the attempt has been made.
func (a *AdaptiveMode) GetAttemptToken(ctx context.Context) (func(error) error, error) {
for {
acquiredToken, waitTryAgain := a.rateLimit.AcquireToken(a.options.RequestCost)
if acquiredToken {
break
}
if a.options.FailOnNoAttemptTokens {
return nil, fmt.Errorf(
"unable to get attempt token, and FailOnNoAttemptTokens enables")
}
if err := sdk.SleepWithContext(ctx, waitTryAgain); err != nil {
return nil, fmt.Errorf("failed to wait for token to be available, %w", err)
}
}
return a.handleResponse, nil
}
func (a *AdaptiveMode) handleResponse(opErr error) error {
throttled := a.throttles.IsErrorThrottle(opErr).Bool()
a.rateLimit.Update(throttled)
return nil
}
package retry
import (
"math"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
)
type adaptiveRateLimit struct {
tokenBucketEnabled bool
smooth float64
beta float64
scaleConstant float64
minFillRate float64
fillRate float64
calculatedRate float64
lastRefilled time.Time
measuredTxRate float64
lastTxRateBucket float64
requestCount int64
lastMaxRate float64
lastThrottleTime time.Time
timeWindow float64
tokenBucket *adaptiveTokenBucket
mu sync.Mutex
}
func newAdaptiveRateLimit() *adaptiveRateLimit {
now := sdk.NowTime()
return &adaptiveRateLimit{
smooth: 0.8,
beta: 0.7,
scaleConstant: 0.4,
minFillRate: 0.5,
lastTxRateBucket: math.Floor(timeFloat64Seconds(now)),
lastThrottleTime: now,
tokenBucket: newAdaptiveTokenBucket(0),
}
}
func (a *adaptiveRateLimit) Enable(v bool) {
a.mu.Lock()
defer a.mu.Unlock()
a.tokenBucketEnabled = v
}
func (a *adaptiveRateLimit) AcquireToken(amount uint) (
tokenAcquired bool, waitTryAgain time.Duration,
) {
a.mu.Lock()
defer a.mu.Unlock()
if !a.tokenBucketEnabled {
return true, 0
}
a.tokenBucketRefill()
available, ok := a.tokenBucket.Retrieve(float64(amount))
if !ok {
waitDur := float64Seconds((float64(amount) - available) / a.fillRate)
return false, waitDur
}
return true, 0
}
func (a *adaptiveRateLimit) Update(throttled bool) {
a.mu.Lock()
defer a.mu.Unlock()
a.updateMeasuredRate()
if throttled {
rateToUse := a.measuredTxRate
if a.tokenBucketEnabled {
rateToUse = math.Min(a.measuredTxRate, a.fillRate)
}
a.lastMaxRate = rateToUse
a.calculateTimeWindow()
a.lastThrottleTime = sdk.NowTime()
a.calculatedRate = a.cubicThrottle(rateToUse)
a.tokenBucketEnabled = true
} else {
a.calculateTimeWindow()
a.calculatedRate = a.cubicSuccess(sdk.NowTime())
}
newRate := math.Min(a.calculatedRate, 2*a.measuredTxRate)
a.tokenBucketUpdateRate(newRate)
}
func (a *adaptiveRateLimit) cubicSuccess(t time.Time) float64 {
dt := secondsFloat64(t.Sub(a.lastThrottleTime))
return (a.scaleConstant * math.Pow(dt-a.timeWindow, 3)) + a.lastMaxRate
}
func (a *adaptiveRateLimit) cubicThrottle(rateToUse float64) float64 {
return rateToUse * a.beta
}
func (a *adaptiveRateLimit) calculateTimeWindow() {
a.timeWindow = math.Pow((a.lastMaxRate*(1.-a.beta))/a.scaleConstant, 1./3.)
}
func (a *adaptiveRateLimit) tokenBucketUpdateRate(newRPS float64) {
a.tokenBucketRefill()
a.fillRate = math.Max(newRPS, a.minFillRate)
a.tokenBucket.Resize(newRPS)
}
func (a *adaptiveRateLimit) updateMeasuredRate() {
now := sdk.NowTime()
timeBucket := math.Floor(timeFloat64Seconds(now)*2.) / 2.
a.requestCount++
if timeBucket > a.lastTxRateBucket {
currentRate := float64(a.requestCount) / (timeBucket - a.lastTxRateBucket)
a.measuredTxRate = (currentRate * a.smooth) + (a.measuredTxRate * (1. - a.smooth))
a.requestCount = 0
a.lastTxRateBucket = timeBucket
}
}
func (a *adaptiveRateLimit) tokenBucketRefill() {
now := sdk.NowTime()
if a.lastRefilled.IsZero() {
a.lastRefilled = now
return
}
fillAmount := secondsFloat64(now.Sub(a.lastRefilled)) * a.fillRate
a.tokenBucket.Refund(fillAmount)
a.lastRefilled = now
}
func float64Seconds(v float64) time.Duration {
return time.Duration(v * float64(time.Second))
}
func secondsFloat64(v time.Duration) float64 {
return float64(v) / float64(time.Second)
}
func timeFloat64Seconds(v time.Time) float64 {
return float64(v.UnixNano()) / float64(time.Second)
}
package retry
import (
"math"
"sync"
)
// adaptiveTokenBucket provides a concurrency safe utility for adding and
// removing tokens from the available token bucket.
type adaptiveTokenBucket struct {
remainingTokens float64
maxCapacity float64
minCapacity float64
mu sync.Mutex
}
// newAdaptiveTokenBucket returns an initialized adaptiveTokenBucket with the
// capacity specified.
func newAdaptiveTokenBucket(i float64) *adaptiveTokenBucket {
return &adaptiveTokenBucket{
remainingTokens: i,
maxCapacity: i,
minCapacity: 1,
}
}
// Retrieve attempts to reduce the available tokens by the amount requested. If
// there are tokens available true will be returned along with the number of
// available tokens remaining. If amount requested is larger than the available
// capacity, false will be returned along with the available capacity. If the
// amount is less than the available capacity, the capacity will be reduced by
// that amount, and the remaining capacity and true will be returned.
func (t *adaptiveTokenBucket) Retrieve(amount float64) (available float64, retrieved bool) {
t.mu.Lock()
defer t.mu.Unlock()
if amount > t.remainingTokens {
return t.remainingTokens, false
}
t.remainingTokens -= amount
return t.remainingTokens, true
}
// Refund returns the amount of tokens back to the available token bucket, up
// to the initial capacity.
func (t *adaptiveTokenBucket) Refund(amount float64) {
t.mu.Lock()
defer t.mu.Unlock()
// Capacity cannot exceed max capacity.
t.remainingTokens = math.Min(t.remainingTokens+amount, t.maxCapacity)
}
// Capacity returns the maximum capacity of tokens that the bucket could
// contain.
func (t *adaptiveTokenBucket) Capacity() float64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.maxCapacity
}
// Remaining returns the number of tokens that remaining in the bucket.
func (t *adaptiveTokenBucket) Remaining() float64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.remainingTokens
}
// Resize adjusts the size of the token bucket. Returns the capacity remaining.
func (t *adaptiveTokenBucket) Resize(size float64) float64 {
t.mu.Lock()
defer t.mu.Unlock()
t.maxCapacity = math.Max(size, t.minCapacity)
// Capacity needs to be capped at max capacity, if max size reduced.
t.remainingTokens = math.Min(t.remainingTokens, t.maxCapacity)
return t.remainingTokens
}
package retry
import (
"context"
"github.com/aws/smithy-go/metrics"
"github.com/aws/smithy-go/middleware"
)
type attemptMetrics struct {
Attempts metrics.Int64Counter
Errors metrics.Int64Counter
AttemptDuration metrics.Float64Histogram
}
func newAttemptMetrics(meter metrics.Meter) (*attemptMetrics, error) {
m := &attemptMetrics{}
var err error
m.Attempts, err = meter.Int64Counter("client.call.attempts", func(o *metrics.InstrumentOptions) {
o.UnitLabel = "{attempt}"
o.Description = "The number of attempts for an individual operation"
})
if err != nil {
return nil, err
}
m.Errors, err = meter.Int64Counter("client.call.errors", func(o *metrics.InstrumentOptions) {
o.UnitLabel = "{error}"
o.Description = "The number of errors for an operation"
})
if err != nil {
return nil, err
}
m.AttemptDuration, err = meter.Float64Histogram("client.call.attempt_duration", func(o *metrics.InstrumentOptions) {
o.UnitLabel = "s"
o.Description = "The time it takes to connect to the service, send the request, and get back HTTP status code and headers (including time queued waiting to be sent)"
})
if err != nil {
return nil, err
}
return m, nil
}
func withOperationMetadata(ctx context.Context) metrics.RecordMetricOption {
return func(o *metrics.RecordMetricOptions) {
o.Properties.Set("rpc.service", middleware.GetServiceID(ctx))
o.Properties.Set("rpc.method", middleware.GetOperationName(ctx))
}
}
// Package retry provides interfaces and implementations for SDK request retry behavior.
//
// # Retryer Interface and Implementations
//
// This package defines Retryer interface that is used to either implement custom retry behavior
// or to extend the existing retry implementations provided by the SDK. This package provides a single
// retry implementation: Standard.
//
// # Standard
//
// Standard is the default retryer implementation used by service clients. The standard retryer is a rate limited
// retryer that has a configurable max attempts to limit the number of retry attempts when a retryable error occurs.
// In addition, the retryer uses a configurable token bucket to rate limit the retry attempts across the client,
// and uses an additional delay policy to limit the time between a requests subsequent attempts.
//
// By default the standard retryer uses the DefaultRetryables slice of IsErrorRetryable types to determine whether
// a given error is retryable. By default this list of retryables includes the following:
// - Retrying errors that implement the RetryableError method, and return true.
// - Connection Errors
// - Errors that implement a ConnectionError, Temporary, or Timeout method that return true.
// - Connection Reset Errors.
// - net.OpErr types that are dialing errors or are temporary.
// - HTTP Status Codes: 500, 502, 503, and 504.
// - API Error Codes
// - RequestTimeout, RequestTimeoutException
// - Throttling, ThrottlingException, ThrottledException, RequestThrottledException, TooManyRequestsException,
// RequestThrottled, SlowDown, EC2ThrottledException
// - ProvisionedThroughputExceededException, RequestLimitExceeded, BandwidthLimitExceeded, LimitExceededException
// - TransactionInProgressException, PriorRequestNotComplete
//
// The standard retryer will not retry a request in the event if the context associated with the request
// has been cancelled. Applications must handle this case explicitly if they wish to retry with a different context
// value.
//
// You can configure the standard retryer implementation to fit your applications by constructing a standard retryer
// using the NewStandard function, and providing one more functional argument that mutate the StandardOptions
// structure. StandardOptions provides the ability to modify the token bucket rate limiter, retryable error conditions,
// and the retry delay policy.
//
// For example to modify the default retry attempts for the standard retryer:
//
// // configure the custom retryer
// customRetry := retry.NewStandard(func(o *retry.StandardOptions) {
// o.MaxAttempts = 5
// })
//
// // create a service client with the retryer
// s3.NewFromConfig(cfg, func(o *s3.Options) {
// o.Retryer = customRetry
// })
//
// # Utilities
//
// A number of package functions have been provided to easily wrap retryer implementations in an implementation agnostic
// way. These are:
//
// AddWithErrorCodes - Provides the ability to add additional API error codes that should be considered retryable
// in addition to those considered retryable by the provided retryer.
//
// AddWithMaxAttempts - Provides the ability to set the max number of attempts for retrying a request by wrapping
// a retryer implementation.
//
// AddWithMaxBackoffDelay - Provides the ability to set the max back off delay that can occur before retrying a
// request by wrapping a retryer implementation.
//
// The following package functions have been provided to easily satisfy different retry interfaces to further customize
// a given retryer's behavior:
//
// BackoffDelayerFunc - Can be used to wrap a function to satisfy the BackoffDelayer interface. For example,
// you can use this method to easily create custom back off policies to be used with the
// standard retryer.
//
// IsErrorRetryableFunc - Can be used to wrap a function to satisfy the IsErrorRetryable interface. For example,
// this can be used to extend the standard retryer to add additional logic to determine if an
// error should be retried.
//
// IsErrorTimeoutFunc - Can be used to wrap a function to satisfy IsErrorTimeout interface. For example,
// this can be used to extend the standard retryer to add additional logic to determine if an
// error should be considered a timeout.
package retry
package retry
import "fmt"
// MaxAttemptsError provides the error when the maximum number of attempts have
// been exceeded.
type MaxAttemptsError struct {
Attempt int
Err error
}
func (e *MaxAttemptsError) Error() string {
return fmt.Sprintf("exceeded maximum number of attempts, %d, %v", e.Attempt, e.Err)
}
// Unwrap returns the nested error causing the max attempts error. Provides the
// implementation for errors.Is and errors.As to unwrap nested errors.
func (e *MaxAttemptsError) Unwrap() error {
return e.Err
}
package retry
import (
"math"
"time"
"github.com/aws/aws-sdk-go-v2/internal/rand"
"github.com/aws/aws-sdk-go-v2/internal/timeconv"
)
// ExponentialJitterBackoff provides backoff delays with jitter based on the
// number of attempts.
type ExponentialJitterBackoff struct {
maxBackoff time.Duration
// precomputed number of attempts needed to reach max backoff.
maxBackoffAttempts float64
randFloat64 func() (float64, error)
}
// NewExponentialJitterBackoff returns an ExponentialJitterBackoff configured
// for the max backoff.
func NewExponentialJitterBackoff(maxBackoff time.Duration) *ExponentialJitterBackoff {
return &ExponentialJitterBackoff{
maxBackoff: maxBackoff,
maxBackoffAttempts: math.Log2(
float64(maxBackoff) / float64(time.Second)),
randFloat64: rand.CryptoRandFloat64,
}
}
// BackoffDelay returns the duration to wait before the next attempt should be
// made. Returns an error if unable get a duration.
func (j *ExponentialJitterBackoff) BackoffDelay(attempt int, err error) (time.Duration, error) {
if attempt > int(j.maxBackoffAttempts) {
return j.maxBackoff, nil
}
b, err := j.randFloat64()
if err != nil {
return 0, err
}
// [0.0, 1.0) * 2 ^ attempts
ri := int64(1 << uint64(attempt))
delaySeconds := b * float64(ri)
return timeconv.FloatSecondsDur(delaySeconds), nil
}
package retry
import (
awsmiddle "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/smithy-go/middleware"
)
// attemptResultsKey is a metadata accessor key to retrieve metadata
// for all request attempts.
type attemptResultsKey struct {
}
// GetAttemptResults retrieves attempts results from middleware metadata.
func GetAttemptResults(metadata middleware.Metadata) (AttemptResults, bool) {
m, ok := metadata.Get(attemptResultsKey{}).(AttemptResults)
return m, ok
}
// AttemptResults represents struct containing metadata returned by all request attempts.
type AttemptResults struct {
// Results is a slice consisting attempt result from all request attempts.
// Results are stored in order request attempt is made.
Results []AttemptResult
}
// AttemptResult represents attempt result returned by a single request attempt.
type AttemptResult struct {
// Err is the error if received for the request attempt.
Err error
// Retryable denotes if request may be retried. This states if an
// error is considered retryable.
Retryable bool
// Retried indicates if this request was retried.
Retried bool
// ResponseMetadata is any existing metadata passed via the response middlewares.
ResponseMetadata middleware.Metadata
}
// addAttemptResults adds attempt results to middleware metadata
func addAttemptResults(metadata *middleware.Metadata, v AttemptResults) {
metadata.Set(attemptResultsKey{}, v)
}
// GetRawResponse returns raw response recorded for the attempt result
func (a AttemptResult) GetRawResponse() interface{} {
return awsmiddle.GetRawResponse(a.ResponseMetadata)
}
package retry
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"time"
internalcontext "github.com/aws/aws-sdk-go-v2/internal/context"
"github.com/aws/smithy-go"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddle "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
"github.com/aws/smithy-go/logging"
"github.com/aws/smithy-go/metrics"
smithymiddle "github.com/aws/smithy-go/middleware"
"github.com/aws/smithy-go/tracing"
"github.com/aws/smithy-go/transport/http"
)
// RequestCloner is a function that can take an input request type and clone
// the request for use in a subsequent retry attempt.
type RequestCloner func(interface{}) interface{}
type retryMetadata struct {
AttemptNum int
AttemptTime time.Time
MaxAttempts int
AttemptClockSkew time.Duration
}
// Attempt is a Smithy Finalize middleware that handles retry attempts using
// the provided Retryer implementation.
type Attempt struct {
// Enable the logging of retry attempts performed by the SDK. This will
// include logging retry attempts, unretryable errors, and when max
// attempts are reached.
LogAttempts bool
// A Meter instance for recording retry-related metrics.
OperationMeter metrics.Meter
retryer aws.RetryerV2
requestCloner RequestCloner
}
// define the threshold at which we will consider certain kind of errors to be probably
// caused by clock skew
const skewThreshold = 4 * time.Minute
// NewAttemptMiddleware returns a new Attempt retry middleware.
func NewAttemptMiddleware(retryer aws.Retryer, requestCloner RequestCloner, optFns ...func(*Attempt)) *Attempt {
m := &Attempt{
retryer: wrapAsRetryerV2(retryer),
requestCloner: requestCloner,
}
for _, fn := range optFns {
fn(m)
}
if m.OperationMeter == nil {
m.OperationMeter = metrics.NopMeterProvider{}.Meter("")
}
return m
}
// ID returns the middleware identifier
func (r *Attempt) ID() string { return "Retry" }
func (r Attempt) logf(logger logging.Logger, classification logging.Classification, format string, v ...interface{}) {
if !r.LogAttempts {
return
}
logger.Logf(classification, format, v...)
}
// HandleFinalize utilizes the provider Retryer implementation to attempt
// retries over the next handler
func (r *Attempt) HandleFinalize(ctx context.Context, in smithymiddle.FinalizeInput, next smithymiddle.FinalizeHandler) (
out smithymiddle.FinalizeOutput, metadata smithymiddle.Metadata, err error,
) {
var attemptNum int
var attemptClockSkew time.Duration
var attemptResults AttemptResults
maxAttempts := r.retryer.MaxAttempts()
releaseRetryToken := nopRelease
retryMetrics, err := newAttemptMetrics(r.OperationMeter)
if err != nil {
return out, metadata, err
}
for {
attemptNum++
attemptInput := in
attemptInput.Request = r.requestCloner(attemptInput.Request)
// Record the metadata for the for attempt being started.
attemptCtx := setRetryMetadata(ctx, retryMetadata{
AttemptNum: attemptNum,
AttemptTime: sdk.NowTime().UTC(),
MaxAttempts: maxAttempts,
AttemptClockSkew: attemptClockSkew,
})
// Setting clock skew to be used on other context (like signing)
ctx = internalcontext.SetAttemptSkewContext(ctx, attemptClockSkew)
var attemptResult AttemptResult
attemptCtx, span := tracing.StartSpan(attemptCtx, "Attempt", func(o *tracing.SpanOptions) {
o.Properties.Set("operation.attempt", attemptNum)
})
retryMetrics.Attempts.Add(ctx, 1, withOperationMetadata(ctx))
start := sdk.NowTime()
out, attemptResult, releaseRetryToken, err = r.handleAttempt(attemptCtx, attemptInput, releaseRetryToken, next)
elapsed := sdk.NowTime().Sub(start)
retryMetrics.AttemptDuration.Record(ctx, float64(elapsed)/1e9, withOperationMetadata(ctx))
if err != nil {
retryMetrics.Errors.Add(ctx, 1, withOperationMetadata(ctx), func(o *metrics.RecordMetricOptions) {
o.Properties.Set("exception.type", errorType(err))
})
}
span.End()
attemptClockSkew, _ = awsmiddle.GetAttemptSkew(attemptResult.ResponseMetadata)
// AttemptResult Retried states that the attempt was not successful, and
// should be retried.
shouldRetry := attemptResult.Retried
// Add attempt metadata to list of all attempt metadata
attemptResults.Results = append(attemptResults.Results, attemptResult)
if !shouldRetry {
// Ensure the last response's metadata is used as the bases for result
// metadata returned by the stack. The Slice of attempt results
// will be added to this cloned metadata.
metadata = attemptResult.ResponseMetadata.Clone()
break
}
}
addAttemptResults(&metadata, attemptResults)
return out, metadata, err
}
// handleAttempt handles an individual request attempt.
func (r *Attempt) handleAttempt(
ctx context.Context, in smithymiddle.FinalizeInput, releaseRetryToken func(error) error, next smithymiddle.FinalizeHandler,
) (
out smithymiddle.FinalizeOutput, attemptResult AttemptResult, _ func(error) error, err error,
) {
defer func() {
attemptResult.Err = err
}()
// Short circuit if this attempt never can succeed because the context is
// canceled. This reduces the chance of token pools being modified for
// attempts that will not be made
select {
case <-ctx.Done():
return out, attemptResult, nopRelease, ctx.Err()
default:
}
//------------------------------
// Get Attempt Token
//------------------------------
releaseAttemptToken, err := r.retryer.GetAttemptToken(ctx)
if err != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to get retry Send token, %w", err)
}
//------------------------------
// Send Attempt
//------------------------------
logger := smithymiddle.GetLogger(ctx)
service, operation := awsmiddle.GetServiceID(ctx), awsmiddle.GetOperationName(ctx)
retryMetadata, _ := getRetryMetadata(ctx)
attemptNum := retryMetadata.AttemptNum
maxAttempts := retryMetadata.MaxAttempts
// Following attempts must ensure the request payload stream starts in a
// rewound state.
if attemptNum > 1 {
if rewindable, ok := in.Request.(interface{ RewindStream() error }); ok {
if rewindErr := rewindable.RewindStream(); rewindErr != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to rewind transport stream for retry, %w", rewindErr)
}
}
r.logf(logger, logging.Debug, "retrying request %s/%s, attempt %d",
service, operation, attemptNum)
}
var metadata smithymiddle.Metadata
out, metadata, err = next.HandleFinalize(ctx, in)
attemptResult.ResponseMetadata = metadata
//------------------------------
// Bookkeeping
//------------------------------
// Release the retry token based on the state of the attempt's error (if any).
if releaseError := releaseRetryToken(err); releaseError != nil && err != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to release retry token after request error, %w", err)
}
// Release the attempt token based on the state of the attempt's error (if any).
if releaseError := releaseAttemptToken(err); releaseError != nil && err != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to release initial token after request error, %w", err)
}
// If there was no error making the attempt, nothing further to do. There
// will be nothing to retry.
if err == nil {
return out, attemptResult, nopRelease, err
}
err = wrapAsClockSkew(ctx, err)
//------------------------------
// Is Retryable and Should Retry
//------------------------------
// If the attempt failed with an unretryable error, nothing further to do
// but return, and inform the caller about the terminal failure.
retryable := r.retryer.IsErrorRetryable(err)
if !retryable {
r.logf(logger, logging.Debug, "request failed with unretryable error %v", err)
return out, attemptResult, nopRelease, err
}
// set retryable to true
attemptResult.Retryable = true
// Once the maximum number of attempts have been exhausted there is nothing
// further to do other than inform the caller about the terminal failure.
if maxAttempts > 0 && attemptNum >= maxAttempts {
r.logf(logger, logging.Debug, "max retry attempts exhausted, max %d", maxAttempts)
err = &MaxAttemptsError{
Attempt: attemptNum,
Err: err,
}
return out, attemptResult, nopRelease, err
}
//------------------------------
// Get Retry (aka Retry Quota) Token
//------------------------------
// Get a retry token that will be released after the
releaseRetryToken, retryTokenErr := r.retryer.GetRetryToken(ctx, err)
if retryTokenErr != nil {
return out, attemptResult, nopRelease, retryTokenErr
}
//------------------------------
// Retry Delay and Sleep
//------------------------------
// Get the retry delay before another attempt can be made, and sleep for
// that time. Potentially early exist if the sleep is canceled via the
// context.
retryDelay, reqErr := r.retryer.RetryDelay(attemptNum, err)
if reqErr != nil {
return out, attemptResult, releaseRetryToken, reqErr
}
if reqErr = sdk.SleepWithContext(ctx, retryDelay); reqErr != nil {
err = &aws.RequestCanceledError{Err: reqErr}
return out, attemptResult, releaseRetryToken, err
}
// The request should be re-attempted.
attemptResult.Retried = true
return out, attemptResult, releaseRetryToken, err
}
// errors that, if detected when we know there's a clock skew,
// can be retried and have a high chance of success
var possibleSkewCodes = map[string]struct{}{
"InvalidSignatureException": {},
"SignatureDoesNotMatch": {},
"AuthFailure": {},
}
var definiteSkewCodes = map[string]struct{}{
"RequestExpired": {},
"RequestInTheFuture": {},
"RequestTimeTooSkewed": {},
}
// wrapAsClockSkew checks if this error could be related to a clock skew
// error and if so, wrap the error.
func wrapAsClockSkew(ctx context.Context, err error) error {
var v interface{ ErrorCode() string }
if !errors.As(err, &v) {
return err
}
if _, ok := definiteSkewCodes[v.ErrorCode()]; ok {
return &retryableClockSkewError{Err: err}
}
_, isPossibleSkewCode := possibleSkewCodes[v.ErrorCode()]
if skew := internalcontext.GetAttemptSkewContext(ctx); skew > skewThreshold && isPossibleSkewCode {
return &retryableClockSkewError{Err: err}
}
return err
}
// MetricsHeader attaches SDK request metric header for retries to the transport
type MetricsHeader struct{}
// ID returns the middleware identifier
func (r *MetricsHeader) ID() string {
return "RetryMetricsHeader"
}
// HandleFinalize attaches the SDK request metric header to the transport layer
func (r MetricsHeader) HandleFinalize(ctx context.Context, in smithymiddle.FinalizeInput, next smithymiddle.FinalizeHandler) (
out smithymiddle.FinalizeOutput, metadata smithymiddle.Metadata, err error,
) {
retryMetadata, _ := getRetryMetadata(ctx)
const retryMetricHeader = "Amz-Sdk-Request"
var parts []string
parts = append(parts, "attempt="+strconv.Itoa(retryMetadata.AttemptNum))
if retryMetadata.MaxAttempts != 0 {
parts = append(parts, "max="+strconv.Itoa(retryMetadata.MaxAttempts))
}
var ttl time.Time
if deadline, ok := ctx.Deadline(); ok {
ttl = deadline
}
// Only append the TTL if it can be determined.
if !ttl.IsZero() && retryMetadata.AttemptClockSkew > 0 {
const unixTimeFormat = "20060102T150405Z"
ttl = ttl.Add(retryMetadata.AttemptClockSkew)
parts = append(parts, "ttl="+ttl.Format(unixTimeFormat))
}
switch req := in.Request.(type) {
case *http.Request:
req.Header[retryMetricHeader] = append(req.Header[retryMetricHeader][:0], strings.Join(parts, "; "))
default:
return out, metadata, fmt.Errorf("unknown transport type %T", req)
}
return next.HandleFinalize(ctx, in)
}
type retryMetadataKey struct{}
// getRetryMetadata retrieves retryMetadata from the context and a bool
// indicating if it was set.
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func getRetryMetadata(ctx context.Context) (metadata retryMetadata, ok bool) {
metadata, ok = smithymiddle.GetStackValue(ctx, retryMetadataKey{}).(retryMetadata)
return metadata, ok
}
// setRetryMetadata sets the retryMetadata on the context.
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func setRetryMetadata(ctx context.Context, metadata retryMetadata) context.Context {
return smithymiddle.WithStackValue(ctx, retryMetadataKey{}, metadata)
}
// AddRetryMiddlewaresOptions is the set of options that can be passed to
// AddRetryMiddlewares for configuring retry associated middleware.
type AddRetryMiddlewaresOptions struct {
Retryer aws.Retryer
// Enable the logging of retry attempts performed by the SDK. This will
// include logging retry attempts, unretryable errors, and when max
// attempts are reached.
LogRetryAttempts bool
}
// AddRetryMiddlewares adds retry middleware to operation middleware stack
func AddRetryMiddlewares(stack *smithymiddle.Stack, options AddRetryMiddlewaresOptions) error {
attempt := NewAttemptMiddleware(options.Retryer, http.RequestCloner, func(middleware *Attempt) {
middleware.LogAttempts = options.LogRetryAttempts
})
// index retry to before signing, if signing exists
if err := stack.Finalize.Insert(attempt, "Signing", smithymiddle.Before); err != nil {
return err
}
if err := stack.Finalize.Insert(&MetricsHeader{}, attempt.ID(), smithymiddle.After); err != nil {
return err
}
return nil
}
// Determines the value of exception.type for metrics purposes. We prefer an
// API-specific error code, otherwise it's just the Go type for the value.
func errorType(err error) string {
var terr smithy.APIError
if errors.As(err, &terr) {
return terr.ErrorCode()
}
return fmt.Sprintf("%T", err)
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
package retry
import (
"errors"
"github.com/aws/aws-sdk-go-v2/aws"
)
// IsErrorTimeout provides the interface of an implementation to determine if
// a error matches.
type IsErrorTimeout interface {
IsErrorTimeout(err error) aws.Ternary
}
// IsErrorTimeouts is a collection of checks to determine of the error is
// retryable. Iterates through the checks and returns the state of retryable
// if any check returns something other than unknown.
type IsErrorTimeouts []IsErrorTimeout
// IsErrorTimeout returns if the error is retryable if any of the checks in
// the list return a value other than unknown.
func (ts IsErrorTimeouts) IsErrorTimeout(err error) aws.Ternary {
for _, t := range ts {
if v := t.IsErrorTimeout(err); v != aws.UnknownTernary {
return v
}
}
return aws.UnknownTernary
}
// IsErrorTimeoutFunc wraps a function with the IsErrorTimeout interface.
type IsErrorTimeoutFunc func(error) aws.Ternary
// IsErrorTimeout returns if the error is retryable.
func (fn IsErrorTimeoutFunc) IsErrorTimeout(err error) aws.Ternary {
return fn(err)
}
// TimeouterError provides the IsErrorTimeout implementation for determining if
// an error is a timeout based on type with the Timeout method.
type TimeouterError struct{}
// IsErrorTimeout returns if the error is a timeout error.
func (t TimeouterError) IsErrorTimeout(err error) aws.Ternary {
var v interface{ Timeout() bool }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
return aws.BoolTernary(v.Timeout())
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment