Special retry handling for watchdog timeouts, which uses its own fail counter.

@return true if a retry has been scheduled
763095
private void handleTimeoutError(Status status) {
    Preconditions.checkArgument(status.getCause() instanceof StreamWaitTimeoutException,
        "status is not caused by a StreamWaitTimeoutException");
    StreamWaitTimeoutException e = ((StreamWaitTimeoutException) status.getCause());

    // Cancel the existing rpc.
    rpcTimerContext.close();
    failedCount++;

    // Can this request be retried
    int maxRetries = retryOptions.getMaxScanTimeoutRetries();
    if (retryOptions.enableRetries() && ++timeoutRetryCount <= maxRetries) {
      LOG.warn("The client could not get a response in %d ms. Retrying the scan.",
          e.getWaitTimeMs());

      resetStatusBasedBackoff();
      performRetry(0);
    } else {
      LOG.warn("The client could not get a response after %d tries, giving up.",
          timeoutRetryCount);
      rpc.getRpcMetrics().markFailure();
      finalizeStats(status);
      setException(getExhaustedRetriesException(status));
    }
  }
1047770
public boolean isCountedDown() throws InterruptedException, KeeperException {
        if (countedDown) return true;
        int count = ByteBuffer.wrap(m_zk.getData(m_path, false, null)).getInt();
        if (count > 0) return false;
        countedDown = true;
        return true;
    }
127903
public boolean shutdown(long timeout, TimeUnit unit) {
    // Guard against double shutdowns (bug 8).
    if (shuttingDown) {
      getLogger().info("Suppressing duplicate attempt to shut down");
      return false;
    }
    shuttingDown = true;
    String baseName = conn.getName();
    conn.setName(baseName + " - SHUTTING DOWN");
    boolean rv = false;
    try {
      // Conditionally wait
      if (timeout > 0) {
        conn.setName(baseName + " - SHUTTING DOWN (waiting)");
        rv = waitForQueues(timeout, unit);
      }
    } finally {
      // But always begin the shutdown sequence
      try {
        conn.setName(baseName + " - SHUTTING DOWN (telling client)");
        conn.shutdown();
        conn.setName(baseName + " - SHUTTING DOWN (informed client)");
        tcService.shutdown();
      } catch (IOException e) {
        getLogger().warn("exception while shutting down", e);
      }
    }
    return rv;
  }
659832
private boolean handleError(C4Error c4err) {
        // If this is a transient error, or if I'm continuous and the error might go away with a change
        // in network (i.e. network down, hostname unknown), then go offline and retry later.
        final boolean isTransient = C4Replicator.mayBeTransient(c4err) ||
            ((c4err.getDomain() == C4Constants.ErrorDomain.WEB_SOCKET) &&
                (c4err.getCode() == C4WebSocketCloseCode.kWebSocketCloseUserTransient));

        final boolean isNetworkDependent = C4Replicator.mayBeNetworkDependent(c4err);
        if (!isTransient && !(config.isContinuous() && isNetworkDependent)) {
            return false; // nope, this is permanent
        }
        if (!config.isContinuous() && retryCount >= MAX_ONE_SHOT_RETRY_COUNT) {
            return false; //too many retries
        }

        clearRepl();

        if (!isTransient) {
            Log.i(DOMAIN, "%s: Network error (%s); will retry when network changes...", this, c4err);
        }
        else {
            // On transient error, retry periodically, with exponential backoff:
            final int delay = retryDelay(++retryCount);
            Log.i(DOMAIN, "%s: Transient error (%s); will retry in %d sec...", this, c4err, delay);
            handler.schedule(
                new Runnable() {
                    @Override
                    public void run() {
                        retry();
                    }
                },
                delay,
                TimeUnit.SECONDS);
        }

        // Also retry when the network changes:
        startReachabilityObserver();
        return true;
    }
396484
def is_retryable(err):
    
    if isinstance(err, ConnectionClosed):
        # NB: only retryable if we're not mid-streaming
        if err.mid_stream:
            return False
        else:
            return True
    elif isinstance(err, socket.error):
        code = err.args[0]
        return code in CONN_CLOSED_ERRORS
    else:
        return False
1300979
def retry(ExceptionToCheck, tries=3, delay=2, backoff=2):
    
    def deco_retry(f):
        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            try_one_last_time = True
            while mtries > 1:
                try:
                    return f(*args, **kwargs)
                    try_one_last_time = False
                    break
                except ExceptionToCheck:
                    six.print_("Retrying in %s seconds" % str(mdelay))
                    time.sleep(mdelay)
                    mtries -= 1
                    mdelay *= backoff
            if try_one_last_time:
                return f(*args, **kwargs)
            return
        return f_retry  # true decorator
    return deco_retry
1171783
public final void retry(final ConnectionState state, final RetryHandler retryHandler, final ConnectionFailHandler failHandler) throws RetryCancelled {
    if (state.retryCount == 0) {
      state.msBeforeRetry = this.startRetryTime;
    }

    state.retryCount++;
    state.uriIndex = state.retryCount % state.uris.length;

    if (this.shouldRetry(state)) {
      if (logger.isDebugEnabled()) {
        logger.debug("Retry {} to send command", state.retryCount);
      }

      if(state.msBeforeRetry > 0) {
        try {
          state.loop.schedule(new Runnable() {
            @Override
            public void run() {
              try {
                retryHandler.doRetry(state);
              } catch (IOException e) {
                failHandler.catchException(e);
              }
            }
          }, state.msBeforeRetry, TimeUnit.MILLISECONDS);
        } catch (IllegalStateException e) {
          failHandler.catchException(
            new IOException(new CancellationException("Etcd client was closed"))
          );
        }
      } else {
        try {
          retryHandler.doRetry(state);
        } catch (IOException e) {
          failHandler.catchException(e);
        }
      }
    } else {
      throw new RetryCancelled();
    }
  }
535820
def Retry(target=None, args=[], kwargs={},
          options={"retry": True, "interval": 1}):
    
    retry = options["retry"]
    interval = options["interval"]

    while True:
        try:
            resp = target(*args, **kwargs)
            # status error
            if resp.code == 200:
                return resp

            _logger.debug("Request got response status: %s"
                          % (resp.code,) + " retry: %s" % (retry,))
        except TimeoutError:
            _logger.debug("Request message is timeout")
            _logger.debug(args)
            _logger.debug(kwargs)

        # register unsuccessful goes here
        # infinity retry
        if retry is True:
            sleep(interval)
            continue

        # no retry
        if retry is False:
            return None

        # retrying
        try:
            retry = retry - 1
            if retry <= 0:
                return None
        except TypeError as e:
            raise e
        sleep(interval)
161156
protected boolean checkLocks ()
    {
        if (_rebootLocks.isEmpty()) {
            return false;
        }

        log.info("Reboot delayed due to outstanding locks", "locks", _rebootLocks.elements());
        broadcast("m.reboot_delayed");
        (_interval = _omgr.newInterval(new Runnable() {
            public void run () {
                doWarning(getWarnings().length, 0);
            }
        })).schedule(60 * 1000);
        return true;
    }
775900
public <T> boolean waitForResource(int res, int timeout) {
        int RETRY_PERIOD = 250;
        int retryNum = timeout / RETRY_PERIOD;
        for (int i = 0; i < retryNum; i++) {
            T View = (T) this.getView(res);
            if (View != null) {
                break;
            }
            if (i == retryNum - 1) {
                return false;
            }
            this.sleep(RETRY_PERIOD);
        }
        return true;
    }
190557
private static boolean isPermanentError(Throwable throwable) {
    Status status = getStatus(throwable);

    switch (status.getCode()) {
      case CANCELLED:
      case UNKNOWN:
      case DEADLINE_EXCEEDED:
      case RESOURCE_EXHAUSTED:
      case INTERNAL:
      case UNAVAILABLE:
      case UNAUTHENTICATED:
        return false;
      default:
        return true;
    }
  }
694656
protected void retryDelay(int attemptCount) {
        if (attemptCount > 0) {
            try {
                Thread.sleep(attemptCount * retryDelay);
            } catch (InterruptedException e) {
                LOG.debug("Failed to sleep: " + e, e);
            }
        }
    }
658012
def _handle_retryable_error(self, e, retry_count):
        """"""
        if isinstance(e, error.RateLimitEnforcedError):
            time.sleep(e.retry_after)
        else:
            time.sleep(self.RETRY_DELAY * (self.RETRY_BACKOFF ** retry_count))
123502
def is_error_retryable(request, exc):
    
    if is_last_attempt(request):
        return False

    return (
        isinstance(exc, RetryableException)
        or IRetryableError.providedBy(exc)
    )
1357462
public void trackFailure() {
        retryTime = Utils.currentTimeMillis() + (long)backoff;
        backoff = Math.min(backoff * params.multiplier, params.maximum);
    }
528851
def failed(self):
        
        try:
            self._validate_parameters()
            self._base_delay = min(
                self._base_delay * self.exp_fac,
                self.delay_max)
            self._update_delay()
        except Exception:
            ic_logger.exception(
                'Unhandled exception trying to calculate a retry timout')
1158802
@Override
    public void retry(JusError error) throws JusError {
        if(errorFilter != null && !errorFilter.apply(error)) {
            throw error;
        }
        mCurrentRetryCount++;
        currentTimeoutMs = (int) (currentTimeoutMs * backoffMultiplier);
        if (!hasAttemptRemaining()) {
            throw error;
        }
    }