// retryTxCockroach runs the transaction and will retry in case of a retryable error.
// https://www.cockroachlabs.com/docs/transactions.html#client-side-transaction-retries
766723
func retryTxCockroach(tx *sql.Tx, stmts func() error) error {
	// Specify that we intend to retry this txn in case of CockroachDB retryable
	// errors.
	if _, err := tx.Exec("SAVEPOINT cockroach_restart"); err != nil {
		return err
	}

	for {
		released := false

		err := stmts()

		if err == nil {
			// RELEASE acts like COMMIT in CockroachDB. We use it since it gives us an
			// opportunity to react to retryable errors, whereas tx.Commit() doesn't.
			released = true
			if _, err = tx.Exec("RELEASE SAVEPOINT cockroach_restart"); err == nil {
				return nil
			}
		}
		// We got an error; let's see if it's a retryable one and, if so, restart. We look
		// for either the standard PG errcode SerializationFailureError:40001 or the Cockroach extension
		// errcode RetriableError:CR000. The Cockroach extension has been removed server-side, but support
		// for it has been left here for now to maintain backwards compatibility.
		pgErr, ok := err.(pgx.PgError)
		if retryable := ok && (pgErr.Code == "CR000" || pgErr.Code == "40001"); !retryable {
			if released {
				err = &AmbiguousCommitError{err}
			}
			return err
		}
		if _, err = tx.Exec("ROLLBACK TO SAVEPOINT cockroach_restart"); err != nil {
			return err
		}
	}
}
232165
func (r *retrier) Retry(ctx context.Context, retry int, req *http.Request, resp *http.Response, err error) (time.Duration, bool, error) {
	// Fail hard on a specific error
	if err == syscall.ECONNREFUSED {
		err = errors.New("Elasticsearch or network down")
		r.logger.Error(err)
		return 0, false, err
	}

	// Stop after n retries
	if retry >= r.retry {
		r.logger.Debugf("Stop retrying after '%d' retries", retry)
		return 0, false, nil
	}

	// Let the backoff strategy decide how long to wait and whether to stop
	r.logger.Debugln("Retry to connect to Elasticsearch")
	wait, stop := r.backoff.Next(retry)
	return wait, stop, nil
}
518972
func RetryAny(retryFns ...RetryFn) RetryFn {
	return func(attempt Attempt) bool {
		for _, fn := range retryFns {
			if fn(attempt) {
				return true
			}
		}
		return false
	}
}
143393
func Retry(ctx context.Context, f func() error) error {
	// container for atomic error value
	type errWrap struct {
		E error
	}

	// Try the function in a goroutine
	var errVal atomic.Value
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)

		delay := time.Duration(0)
		for {
			// If our context ended, we want to exit right away.
			select {
			case <-ctx.Done():
				return
			case <-time.After(delay):
			}

			// Try the function call
			err := f()

			// return if we have no error, or a FatalError
			done := false
			switch e := err.(type) {
			case nil:
				done = true
			case Fatal:
				err = e.FatalError()
				done = true
			}

			errVal.Store(errWrap{err})

			if done {
				return
			}

			log.Printf("[WARN] retryable error: %v", err)

			delay *= 2

			if delay == 0 {
				delay = initialBackoffDelay
			}

			if delay > maxBackoffDelay {
				delay = maxBackoffDelay
			}

			log.Printf("[INFO] sleeping for %s", delay)
		}
	}()

	// Wait for completion
	select {
	case <-ctx.Done():
	case <-doneCh:
	}

	var lastErr error
	// Check if we got an error executing
	if ev, ok := errVal.Load().(errWrap); ok {
		lastErr = ev.E
	}

	// Check if we have a context error to check if we're interrupted or timeout
	switch ctx.Err() {
	case context.Canceled:
		return fmt.Errorf("interrupted - last error: %v", lastErr)
	case context.DeadlineExceeded:
		return fmt.Errorf("timeout - last error: %v", lastErr)
	}

	if lastErr != nil {
		return lastErr
	}
	return nil
}
1186329
func (f *Flock) TryLockContext(ctx context.Context, retryDelay time.Duration) (bool, error) {
	return tryCtx(ctx, f.TryLock, retryDelay)
}
789086
func retry(op Op, times int) Op {
	return func(req rets.Requester, urls rets.CapabilityURLs) error {
		var err error
		for ; times > 0; times-- {
			err = op(req, urls)
			if err == nil {
				return nil
			}
		}
		return err
	}
}
694456
func (sm *SyncManager) QueueTx(tx *btcutil.Tx, peer *peerpkg.Peer, done chan struct{}) {
	// Don't accept more transactions if we're shutting down.
	if atomic.LoadInt32(&sm.shutdown) != 0 {
		done <- struct{}{}
		return
	}

	sm.msgChan <- &txMsg{tx: tx, peer: peer, reply: done}
}
1291752
func teamEKRetryWrapper(mctx libkb.MetaContext, retryFn func() error) (err error) {
	knownRaceConditions := []keybase1.StatusCode{
		keybase1.StatusCode_SCSigWrongKey,
		keybase1.StatusCode_SCSigOldSeqno,
		keybase1.StatusCode_SCEphemeralKeyBadGeneration,
		keybase1.StatusCode_SCEphemeralKeyUnexpectedBox,
		keybase1.StatusCode_SCEphemeralKeyMissingBox,
		keybase1.StatusCode_SCEphemeralKeyWrongNumberOfKeys,
	}
	for tries := 0; tries < maxRetries; tries++ {
		if err = retryFn(); err == nil {
			return nil
		}
		retryableError := false
		for _, code := range knownRaceConditions {
			if libkb.IsAppStatusCode(err, code) {
				mctx.Debug("teamEKRetryWrapper found a retryable error on try %d: %s", tries, err)
				retryableError = true
				break
			}
		}
		if !retryableError {
			return err
		}
	}
	return nil
}
460812
func (rc *Context) Retry(err error) {
	rc.LastError = err

	if rc.Settings.Consumer != nil {
		rc.Settings.Consumer.PauseProgress()
		if neterr.IsNetworkError(err) {
			rc.Settings.Consumer.Infof("having network troubles...")
		} else {
			rc.Settings.Consumer.Infof("%v", err)
		}
	}

	// exponential backoff: 1, 2, 4, 8 seconds...
	delay := int(math.Pow(2, float64(rc.Tries)))
	// ...plus a random number of milliseconds.
	// see https://cloud.google.com/storage/docs/exponential-backoff
	jitter := rand.Int() % 1000

	if rc.Settings.Consumer != nil {
		rc.Settings.Consumer.Infof("Sleeping %d seconds then retrying", delay)
	}

	sleepDuration := time.Second*time.Duration(delay) + time.Millisecond*time.Duration(jitter)
	if rc.Settings.NoSleep {
		if rc.Settings.FakeSleep != nil {
			rc.Settings.FakeSleep(sleepDuration)
		}
	} else {
		time.Sleep(sleepDuration)
	}

	rc.Tries++

	if rc.Settings.Consumer != nil {
		rc.Settings.Consumer.ResumeProgress()
	}
}
122315
func DoWithRetry(f func() error) error {
	var err error
	try := true
	handler := &retryHandler{}
	for try {
		try, err = handler.Try(f)
	}
	return err
}
1181657
func (s *Scorch) RollbackPoints() ([]*RollbackPoint, error) {
	if s.rootBolt == nil {
		return nil, fmt.Errorf("RollbackPoints: root is nil")
	}

	// start a read-only bolt transaction
	tx, err := s.rootBolt.Begin(false)
	if err != nil {
		return nil, fmt.Errorf("RollbackPoints: failed to start" +
			" read-only transaction")
	}

	// read-only bolt transactions to be rolled back
	defer func() {
		_ = tx.Rollback()
	}()

	snapshots := tx.Bucket(boltSnapshotsBucket)
	if snapshots == nil {
		return nil, nil
	}

	rollbackPoints := []*RollbackPoint{}

	c1 := snapshots.Cursor()
	for k, _ := c1.Last(); k != nil; k, _ = c1.Prev() {
		_, snapshotEpoch, err := segment.DecodeUvarintAscending(k)
		if err != nil {
			log.Printf("RollbackPoints:"+
				" unable to parse segment epoch %x, continuing", k)
			continue
		}

		snapshot := snapshots.Bucket(k)
		if snapshot == nil {
			log.Printf("RollbackPoints:"+
				" snapshot key, but bucket missing %x, continuing", k)
			continue
		}

		meta := map[string][]byte{}
		c2 := snapshot.Cursor()
		for j, _ := c2.First(); j != nil; j, _ = c2.Next() {
			if j[0] == boltInternalKey[0] {
				internalBucket := snapshot.Bucket(j)
				err = internalBucket.ForEach(func(key []byte, val []byte) error {
					copiedVal := append([]byte(nil), val...)
					meta[string(key)] = copiedVal
					return nil
				})
				if err != nil {
					break
				}
			}
		}

		if err != nil {
			log.Printf("RollbackPoints:"+
				" failed in fetching internal data: %v", err)
			continue
		}

		rollbackPoints = append(rollbackPoints, &RollbackPoint{
			epoch: snapshotEpoch,
			meta:  meta,
		})
	}

	return rollbackPoints, nil
}
455479
func retryOnError(err error) error {
	switch v := err.(type) {
	case errcode.Errors:
		if len(v) != 0 {
			return retryOnError(v[0])
		}
	case errcode.Error:
		switch v.Code {
		case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeUnsupported, errcode.ErrorCodeDenied, errcode.ErrorCodeTooManyRequests, v2.ErrorCodeNameUnknown:
			return xfer.DoNotRetry{Err: err}
		}
	case *url.Error:
		switch v.Err {
		case auth.ErrNoBasicAuthCredentials, auth.ErrNoToken:
			return xfer.DoNotRetry{Err: v.Err}
		}
		return retryOnError(v.Err)
	case *client.UnexpectedHTTPResponseError:
		return xfer.DoNotRetry{Err: err}
	case error:
		if err == distribution.ErrBlobUnknown {
			return xfer.DoNotRetry{Err: err}
		}
		if strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) {
			return xfer.DoNotRetry{Err: err}
		}
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return err
}
872065
func (cd *ConnectionDetails) RetryLimit() int {
	i, err := strconv.Atoi(defaults.String(cd.Options["retry_limit"], "1000"))
	if err != nil {
		return 100
	}
	return i
}
1174856
func (f *Request) Retry(r int) *Request {
	f.retry = r
	return f
}
216118
func retry(ctx context.Context, f func() bool, delay time.Duration) error {
	if f() {
		return nil
	}

	ticker := time.NewTicker(delay)
	errChan := make(chan error)

	go func() {
		defer close(errChan)

		for {
			select {
			case <-ctx.Done():
				errChan <- ctx.Err()
				return
			case <-ticker.C:
				if f() {
					return
				}
			}
		}
	}()

	return <-errChan
}
1352570
func RetryUntilSucceed(action func() error, desc string, ctx context.Context, timeout time.Duration) {
	ctxCompleted := false
	stop := make(chan bool)
	defer close(stop)
	go func() {
		select {
		case <-ctx.Done():
			ctxCompleted = true
		case <-stop:
		}
	}()
	for {
		log.Debugf("Start %s", desc)
		err := action()
		if err == nil {
			log.Debugf("Completed %s", desc)
			return
		}
		if ctxCompleted {
			log.Debugf("Stop retrying %s", desc)
			return
		}
		log.Debugf("Failed to %s: %+v, retrying in %v", desc, err, timeout)
		time.Sleep(timeout)

	}
}
1185518
func (c *Client) Reindex(sourceIndex, targetIndex string) *Reindexer {
	return NewReindexer(c, sourceIndex, CopyToTargetIndex(targetIndex))
}