// agencyConnectionFailureBackoff returns a backoff delay for cases where all
// agents responded with a non-fatal error.
777301
func agencyConnectionFailureBackoff(lastDelay time.Duration) time.Duration {
	return increaseDelay(lastDelay, 1.5, time.Millisecond, time.Second*2)
}
1061315
func (f *connFailure) Error() string {
	return fmt.Sprintf("connection with %s failed with code=%s",
		f.ID, f.Code,
	)
}
753308
func (b *blockServerRemoteClientHandler) OnConnectError(err error, wait time.Duration) {
	b.log.Warning("%s: connection error: %v; retrying in %s", b.name, err, wait)
	if b.authToken != nil {
		b.authToken.Shutdown()
	}
	b.pinger.cancelTicker()
	// TODO: it might make sense to show something to the user if this is
	// due to authentication, for example.
}
109706
func RetryWithBackoff(backoff wait.Backoff, condition func() (bool, error)) (bool, error) {
	duration := backoff.Duration
	i := 0
	for {
		if i != 0 {
			adjusted := duration
			if backoff.Jitter > 0.0 {
				adjusted = wait.Jitter(duration, backoff.Jitter)
			}
			time.Sleep(adjusted)
			duration = time.Duration(float64(duration) * backoff.Factor)
		}

		i++

		done, err := condition()
		if done {
			return done, err
		}
		noMoreRetries := i >= backoff.Steps
		if !noMoreRetries && err != nil {
			glog.V(2).Infof("retrying after error %v", err)
		}

		if noMoreRetries {
			glog.Infof("hit maximum retries %d with error %v", i, err)
			return done, err
		}
	}
}
173608
func exponentialBackoff(ctx context.Context, req Request, attempts int) (time.Duration, error) {
	if attempts == 0 {
		return time.Duration(0), nil
	}
	return time.Duration(math.Pow(10, float64(attempts))) * time.Millisecond, nil
}
963513
func NewAgency(conn driver.Connection) (Agency, error) {
	return &agency{
		conn: conn,
	}, nil
}
398455
func (e *ExponentialStrategy) Backoff() backoff.Backoff {
	return &exponentialBackoff{
		first: e.opts.first,
		max:   e.opts.max.Nanoseconds(),
		rand:  e.opts.newRand(),
	}
}
1007069
func (f *FSMContext) FailWorkflow(data interface{}, details *string) Outcome {
	d := &swf.Decision{
		DecisionType: S(swf.DecisionTypeFailWorkflowExecution),
		FailWorkflowExecutionDecisionAttributes: &swf.FailWorkflowExecutionDecisionAttributes{
			Details: details,
		},
	}
	return Outcome{
		State:     FailedState,
		Data:      data,
		Decisions: f.Decision(d),
	}
}
555540
func Fallback(h http.Handler) CircuitBreakerOption {
	return func(c *CircuitBreaker) error {
		c.fallback = h
		return nil
	}
}
1005587
func Backoff(base, max, jitter, factor float64, retries int) time.Duration {
	if retries == 0 {
		return 0
	}

	backoff, max := float64(base), float64(max)
	for backoff < max && retries > 0 {
		backoff *= factor
		retries--
	}
	if backoff > max {
		backoff = max
	}

	// Randomize backoff delays so that if a cluster of requests start at
	// the same time, they won't operate in lockstep.
	backoff *= 1 + jitter*(rand.Float64()*2-1)
	if backoff < 0 {
		return 0
	}

	return time.Duration(backoff)
}
578570
func (c *agencyConnection) Endpoints() []string {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	var result []string
	for _, x := range c.connections {
		result = append(result, x.Endpoints()...)
	}
	return result
}
105971
func (bc Config) Backoff(retries int) time.Duration {
	if retries == 0 {
		return bc.BaseDelay
	}
	backoff, max := float64(bc.BaseDelay), float64(bc.MaxDelay)
	for backoff < max && retries > 0 {
		backoff *= bc.Factor
		retries--
	}
	if backoff > max {
		backoff = max
	}
	// Randomize backoff delays so that if a cluster of requests start at
	// the same time, they won't operate in lockstep.
	backoff *= 1 + bc.Jitter*(rand.Float64()*2-1)
	if backoff < 0 {
		return 0
	}
	return time.Duration(backoff)
}
970959
func (r *Request) BackOff(manager BackoffManager) *Request {
	if manager == nil {
		r.backoffMgr = &NoBackoff{}
		return r
	}

	r.backoffMgr = manager
	return r
}
444629
func (b *ExponentialBackoff) Failure(event Event, err error) bool {
	atomic.AddUint64(&b.failures, 1)
	return false
}
346239
func (b *Backoffer) Backoff(typ backoffType, err error) error {
	if strings.Contains(err.Error(), mismatchClusterID) {
		logutil.Logger(context.Background()).Fatal("critical error", zap.Error(err))
	}
	select {
	case <-b.ctx.Done():
		return errors.Trace(err)
	default:
	}

	typ.Counter().Inc()
	// Lazy initialize.
	if b.fn == nil {
		b.fn = make(map[backoffType]func(context.Context) int)
	}
	f, ok := b.fn[typ]
	if !ok {
		f = typ.createFn(b.vars)
		b.fn[typ] = f
	}

	b.totalSleep += f(b.ctx)
	b.types = append(b.types, typ)

	var startTs interface{}
	if ts := b.ctx.Value(txnStartKey); ts != nil {
		startTs = ts
	}
	logutil.Logger(context.Background()).Debug("retry later",
		zap.Error(err),
		zap.Int("totalSleep", b.totalSleep),
		zap.Int("maxSleep", b.maxSleep),
		zap.Stringer("type", typ),
		zap.Reflect("txnStartTS", startTs))

	b.errors = append(b.errors, errors.Errorf("%s at %s", err.Error(), time.Now().Format(time.RFC3339Nano)))
	if b.maxSleep > 0 && b.totalSleep >= b.maxSleep {
		errMsg := fmt.Sprintf("%s backoffer.maxSleep %dms is exceeded, errors:", typ.String(), b.maxSleep)
		for i, err := range b.errors {
			// Print only last 3 errors for non-DEBUG log levels.
			if log.GetLevel() == zapcore.DebugLevel || i >= len(b.errors)-3 {
				errMsg += "\n" + err.Error()
			}
		}
		logutil.Logger(context.Background()).Warn(errMsg)
		// Use the first backoff type to generate a MySQL error.
		return b.types[0].TError()
	}
	return nil
}
691397
func (db *DB) DisableConnFail() {
	db.mu.Lock()
	defer db.mu.Unlock()
	db.isConnFail = false
}
627355
func (db *DB) EnableConnFail() {
	db.mu.Lock()
	defer db.mu.Unlock()
	db.isConnFail = true
}