Put this cluster in retry-wait (or consider it dead)
518737
def try_again_later(self, seconds):
        """"""
        if not self.failed:
            self.fails += 1
            self.retry_at = (dt.datetime.now() + timedelta(seconds=seconds))
288813
def wait(self):
    ""
    data=self.getcon().recv(256) # this can raise socket.timeout
    if not data: raise PubsubDisco
    if self.reset:
      self.reset=False # i.e. ack it. reset is used to tell the wait-thread there was a reconnect (though it's plausible that this never happens)
      raise PubsubDisco
    self.buf+=data
    msg,self.buf=complete_message(self.buf)
    return msg
703761
@FFDCIgnore({ InterruptedException.class })
    public synchronized void waitForLevel() {
        // Don't bother waiting if the vm has been shutdown in the meanwhile...
        while (levelReached.get() == false && !shutdownHook.vmShutdown()) {
            try {
                // waiting for a start level event should not take long
                wait(1000);
            } catch (InterruptedException e) {
                /** No-op **/
            }
        }
    }
5684
def start(self, timeout=300):
        """"""
        if self.is_alive:
            return True
        try:
            dbpath = self.cfg.get('dbpath')
            if dbpath and self._is_locked:
                # repair if needed
                logger.info("Performing repair on locked dbpath %s", dbpath)
                process.repair_mongo(self.name, self.cfg['dbpath'])

            self.proc, self.hostname = process.mprocess(
                self.name, self.config_path, self.cfg.get('port', None),
                timeout, self.silence_stdout)
            self.pid = self.proc.pid
            logger.debug("pid={pid}, hostname={hostname}".format(pid=self.pid, hostname=self.hostname))
            self.host = self.hostname.split(':')[0]
            self.port = int(self.hostname.split(':')[1])

            # Wait for Server to respond to isMaster.
            # Only try 6 times, each ConnectionFailure is 30 seconds.
            max_attempts = 6
            for i in range(max_attempts):
                try:
                    self.run_command('isMaster')
                    break
                except pymongo.errors.ConnectionFailure:
                    logger.exception('isMaster command failed:')
            else:
                raise TimeoutError(
                    "Server did not respond to 'isMaster' after %d attempts."
                    % max_attempts)
        except (OSError, TimeoutError):
            logpath = self.cfg.get('logpath')
            if logpath:
                # Copy the server logs into the mongo-orchestration logs.
                logger.error(
                    "Could not start Server. Please find server log below.\n"
                    "=====================================================")
                with open(logpath) as lp:
                    logger.error(lp.read())
            else:
                logger.exception(
                    'Could not start Server, and no logpath was provided!')
            reraise(TimeoutError,
                    'Could not start Server. '
                    'Please check server log located in ' +
                    self.cfg.get('logpath', '<no logpath given>') +
                    ' or the mongo-orchestration log in ' +
                    LOG_FILE + ' for more details.')
        if self.restart_required:
            if self.login:
                # Add users to the appropriate database.
                self._add_users()
            self.stop()

            # Restart with keyfile and auth.
            if self.is_mongos:
                self.config_path, self.cfg = self.__init_mongos(self.cfg)
            else:
                # Add auth options to this Server's config file.
                self.config_path, self.cfg = self.__init_mongod(
                    self.cfg, add_auth=True)
            self.restart_required = False
            self.start()

        return True
738555
function fork() {
            cluster.fork().on('message', function(message) {
                if (message.cmd === 'restart' && message.pid && restartQueue.indexOf(message.pid) === -1) {
                    // When worker asks to restart gracefully in cluster, then add it to restart queue.
                    log('Cluster graceful shutdown: queued. pid=' + message.pid);
                    restartQueue.push(message.pid);
                    checkRestartQueue();
                }
            });
        }
543120
def _should_retry(resp):
  """"""
  return (resp.status_code == httplib.REQUEST_TIMEOUT or
          (resp.status_code >= 500 and
           resp.status_code < 600))
583800
def wait_for_healthy(
        raiden: 'RaidenService',
        node_address: Address,
        retry_timeout: float,
) -> None:
    
    network_statuses = views.get_networkstatuses(
        views.state_from_raiden(raiden),
    )

    while network_statuses.get(node_address) != NODE_NETWORK_REACHABLE:
        gevent.sleep(retry_timeout)
        network_statuses = views.get_networkstatuses(
            views.state_from_raiden(raiden),
        )
449957
public void waitForRestart() {
    if (standbyThread != null) {
      try {
        // if this is the standby avatarnode, then wait for the Standby to exit
        standbyThread.join();
      } catch (InterruptedException ie) {
        //eat it up
      }
      standbyThread = null;
      LOG.info("waitForRestart: Standby thread exited.");

      InjectionHandler.processEvent(InjectionEvent.AVATARNODE_WAIT_FOR_RESTART);
      while (failoverState == FailoverState.START_FAILOVER
          || failoverState == FailoverState.AWAIT_FAILOVER) {
        LOG.info("Current state : " + failoverState
            + ". Waiting for failover ....");
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ie) {
          throw new RuntimeException("waitForRestart() interrupted");
        }
      }

      // if we are still in standbymode, that means we need to restart from
      // scratch.
      if (getAvatar() == Avatar.STANDBY) {
        runInfo.isRunning = false;
        LOG.info("waitForRestart Stopping encapsulated namenode.");
        super.stop(); // terminate encapsulated namenode
        super.join(); // wait for encapsulated namenode to exit
        shutdownStandby();
        LOG.info("waitForRestart exiting");
        return;
      }
    }
    super.join();            // wait for encapsulated namenode
  }
458915
public synchronized void sendRetry(long retry, EventCallback callback) {

        if (open == 0 || shutdown) {
            if (callback != null) {
                callback.failed(this, null, null, null, new ClosedChannelException());
            }
            return;
        }
        queue.add(new SSEData(retry, callback));
        sink.getIoThread().execute(new Runnable() {
            @Override
            public void run() {
                synchronized (ServerSentEventConnection.this) {
                    if (pooled == null) {
                        fillBuffer();
                        writeListener.handleEvent(sink);
                    }
                }
            }
        });
    }
101160
def wait_for_sshd(cls, vm_id):
        """"""
        cls.echo('Waiting for the vm to come online')
        version, ip_addr = cls.vm_ip(vm_id)
        give_up = time.time() + 300
        last_error = None
        while time.time() < give_up:
            try:
                inet = socket.AF_INET
                if version == 6:
                    inet = socket.AF_INET6
                sd = socket.socket(inet, socket.SOCK_STREAM,
                                   socket.IPPROTO_TCP)
                sd.settimeout(5)
                sd.connect((ip_addr, 22))
                sd.recv(1024)
                return
            except socket.error as err:
                if err.errno == errno.EHOSTUNREACH and version == 6:
                    cls.error('%s is not reachable, you may be missing '
                              'IPv6 connectivity' % ip_addr)
                last_error = err
                time.sleep(1)
            except Exception as err:
                last_error = err
                time.sleep(1)
        cls.error('VM did not spin up (last error: %s)' % last_error)
1203743
private void waitForJournalFlush() throws UnavailableException {
    if (mFlushCounter == INVALID_FLUSH_COUNTER) {
      // Check this before the precondition.
      return;
    }

    RetryPolicy retry = new TimeoutRetry(FLUSH_RETRY_TIMEOUT_MS, FLUSH_RETRY_INTERVAL_MS);
    while (retry.attempt()) {
      try {
        mAsyncJournalWriter.flush(mFlushCounter);
        return;
      } catch (IOException e) {
        LOG.warn("Journal flush failed. retrying...", e);
      } catch (JournalClosedException e) {
        throw new UnavailableException(String.format("Failed to complete request: %s",
            e.getMessage()), e);
      } catch (Throwable e) {
        ProcessUtils.fatalError(LOG, e, "Journal flush failed");
      }
    }
    ProcessUtils.fatalError(LOG, "Journal flush failed after %d attempts", retry.getAttemptCount());
  }
997573
def retry(self, **kwargs):
        
        path = '%s/%s/retry' % (self.manager.path, self.get_id())
        self.manager.gitlab.http_post(path)
178973
def wait_for_instance_death(instance, opts={})
      wait_for_instance_status(:terminated, instance, opts) if instance.exists?
    rescue AWS::EC2::Errors::InvalidInstanceID::NotFound
    end
933819
def wait_for(self, ports, interval=1.0, timeout=30, to_start=True):
        
        threads = []
        queue = Queue.Queue()

        for port in ports:
            threads.append(threading.Thread(target=wait_for_host, args=(
                port, interval, timeout, to_start, queue,
                self.ssl_pymongo_options)))

        if self.args and 'verbose' in self.args and self.args['verbose']:
            print("waiting for nodes %s..."
                  % ('to start' if to_start else 'to shutdown'))

        for thread in threads:
            thread.start()

        for thread in threads:
            thread.join()

        # get all results back and return tuple
        return tuple(queue.get_nowait() for _ in ports)
484467
def wait( self, timeout = -1 ):
        """"""

        # we can't use ipyparallel.Client.wait() for this, because that
        # method only works for cases where the Client object is the one that
        # submitted the jobs to the cluster hub -- and therefore has the
        # necessary data structures to perform synchronisation. This isn't the
        # case for us, as one of the main goals of epyc is to support disconnected
        # operation, which implies a different Client object retrieving results
        # than the one that submitted the jobs in the first place. This is
        # unfortunate, but understandable given the typical use cases for
        # Client objects.
        #
        # Instead. we have to code around a little busily. The ClusterLab.WaitingTime
        # global sets the latency for waiting, and we repeatedly wait for this amount
        # of time before updating the results. The latency value essentially controls
        # how busy this process is: given that most simulations are expected to
        # be long, a latency in the tens of seconds feels about right as a default
        if self.numberOfPendingResults() > 0:
            # we've got pending results, wait for them
            timeWaited = 0
            while (timeout < 0) or (timeWaited < timeout):
                if self.numberOfPendingResults() == 0:
                    # no pending jobs left, we're complete
                    return True
                else:
                    # not done yet, calculate the waiting period
                    if timeout == -1:
                        # wait for the default waiting period
                        dt = self.WaitingTime
                    else:
                        # wait for the default waiting period or until the end of the timeout.
                        # whichever comes first
                        if (timeout - timeWaited) < self.WaitingTime:
                            dt = timeout - timeWaited
                        else:
                            dt = self.WaitingTime
                            
                    # sleep for a while
                    time.sleep(dt)
                    timeWaited = timeWaited + dt

            # if we get here, the timeout expired, so do a final check
            # and then exit
            return (self.numberOfPendingResults() == 0)

        else:
            # no results, so we got them all
            return True
1110528
def _wait_for_result(self):
        """"""
        basetime = 0.018 if self._low_res else 0.128
        sleep(basetime * (self._mtreg / 69.0) + self._delay)
1230304
def wait(self):
        
        try:
            SpawningProxy(self.containers, abort_on_error=True).wait()
        except Exception:
            # If a single container failed, stop its peers and re-raise the
            # exception
            self.stop()
            raise