Reads and locks the next_request_to_process for a number of flows.
354554
def _ReadAndLockNextRequestsToProcess(self, flow_keys, cursor):
    """"""

    query = """
      SELECT client_id, flow_id, next_request_to_process
      FROM flows
      WHERE {conditions}
      FOR UPDATE
    """
    condition_template = "(client_id = %s AND flow_id = %s)"
    conditions = [condition_template] * len(flow_keys)
    query = query.format(conditions=" OR ".join(conditions))
    args = []
    for client_id, flow_id in flow_keys:
      args.append(db_utils.ClientIDToInt(client_id))
      args.append(db_utils.FlowIDToInt(flow_id))

    cursor.execute(query, args)
    next_requests = {}
    for client_id_int, flow_id_int, next_request in cursor.fetchall():
      flow_key = (db_utils.IntToClientID(client_id_int),
                  db_utils.IntToFlowID(flow_id_int))
      next_requests[flow_key] = next_request
    return next_requests
114909
def WriteFlowRequests(self, requests, cursor=None):
    """"""
    args = []
    templates = []
    flow_keys = []
    needs_processing = {}

    for r in requests:
      if r.needs_processing:
        needs_processing.setdefault((r.client_id, r.flow_id), []).append(r)

      flow_keys.append((r.client_id, r.flow_id))
      templates.append("(%s, %s, %s, %s, %s)")
      args.extend([
          db_utils.ClientIDToInt(r.client_id),
          db_utils.FlowIDToInt(r.flow_id), r.request_id, r.needs_processing,
          r.SerializeToString()
      ])

    if needs_processing:
      flow_processing_requests = []
      nr_conditions = []
      nr_args = []
      for client_id, flow_id in needs_processing:
        nr_conditions.append("(client_id=%s AND flow_id=%s)")
        nr_args.append(db_utils.ClientIDToInt(client_id))
        nr_args.append(db_utils.FlowIDToInt(flow_id))

      nr_query = ("SELECT client_id, flow_id, next_request_to_process "
                  "FROM flows WHERE ")
      nr_query += " OR ".join(nr_conditions)

      cursor.execute(nr_query, nr_args)

      db_result = cursor.fetchall()
      for client_id_int, flow_id_int, next_request_to_process in db_result:
        client_id = db_utils.IntToClientID(client_id_int)
        flow_id = db_utils.IntToFlowID(flow_id_int)
        candidate_requests = needs_processing.get((client_id, flow_id), [])
        for r in candidate_requests:
          if next_request_to_process == r.request_id:
            flow_processing_requests.append(
                rdf_flows.FlowProcessingRequest(
                    client_id=client_id,
                    flow_id=flow_id,
                    delivery_time=r.start_time))

      if flow_processing_requests:
        self._WriteFlowProcessingRequests(flow_processing_requests, cursor)

    query = ("INSERT INTO flow_requests "
             "(client_id, flow_id, request_id, needs_processing, request) "
             "VALUES ")
    query += ", ".join(templates)
    try:
      cursor.execute(query, args)
    except MySQLdb.IntegrityError as e:
      raise db.AtLeastOneUnknownFlowError(flow_keys, cause=e)
278251
def process_request_thread(self, request, client_address):
        
        try:
            self.finish_request(request, client_address)
            self.shutdown_request(request)
        except:
            self.handle_error(request, client_address)
            self.shutdown_request(request)
436027
def interprocess_locked(path):
    """"""

    lock = InterProcessLock(path)

    def decorator(f):

        @six.wraps(f)
        def wrapper(*args, **kwargs):
            with lock:
                return f(*args, **kwargs)

        return wrapper

    return decorator
79550
def _ReadFlowResponseCounts(self, request_keys, cursor=None):
    """"""

    query = """
      SELECT
        flow_requests.client_id, flow_requests.flow_id,
        flow_requests.request_id, COUNT(*)
      FROM flow_responses, flow_requests
      WHERE ({conditions}) AND
        flow_requests.client_id = flow_responses.client_id AND
        flow_requests.flow_id = flow_responses.flow_id AND
        flow_requests.request_id = flow_responses.request_id AND
        flow_requests.needs_processing = FALSE
      GROUP BY
        flow_requests.client_id,
        flow_requests.flow_id,
        flow_requests.request_id
    """

    condition_template = """
      (flow_requests.client_id=%s AND
       flow_requests.flow_id=%s AND
       flow_requests.request_id=%s)"""
    conditions = [condition_template] * len(request_keys)
    args = []
    for client_id, flow_id, request_id in request_keys:
      args.append(db_utils.ClientIDToInt(client_id))
      args.append(db_utils.FlowIDToInt(flow_id))
      args.append(request_id)

    query = query.format(conditions=" OR ".join(conditions))
    cursor.execute(query, args)
    response_counts = {}
    for (client_id_int, flow_id_int, request_id, count) in cursor.fetchall():
      request_key = (db_utils.IntToClientID(client_id_int),
                     db_utils.IntToFlowID(flow_id_int), request_id)
      response_counts[request_key] = count
    return response_counts
1010436
def _finish_pending_requests(self) -> None:
        
        while True:
            num_q, ok_list, err_list = self._multi.info_read()
            for curl in ok_list:
                self._finish(curl)
            for curl, errnum, errmsg in err_list:
                self._finish(curl, errnum, errmsg)
            if num_q == 0:
                break
        self._process_queue()
623088
def ReadAllFlowObjects(
      self,
      client_id = None,
      min_create_time = None,
      max_create_time = None,
      include_child_flows = True,
  ):
    """"""
    res = []
    for flow in itervalues(self.flows):
      if ((client_id is None or flow.client_id == client_id) and
          (min_create_time is None or flow.create_time >= min_create_time) and
          (max_create_time is None or flow.create_time <= max_create_time) and
          (include_child_flows or not flow.parent_flow_id)):
        res.append(flow.Copy())
    return res
902158
def ReadFlowRequestsReadyForProcessing(self,
                                         client_id,
                                         flow_id,
                                         next_needed_request,
                                         cursor=None):
    """"""
    query = ("SELECT request, needs_processing, responses_expected, "
             "UNIX_TIMESTAMP(timestamp) "
             "FROM flow_requests "
             "WHERE client_id=%s AND flow_id=%s")
    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]
    cursor.execute(query, args)

    requests = {}
    for req, needs_processing, responses_expected, ts in cursor.fetchall():
      if not needs_processing:
        continue

      request = rdf_flow_objects.FlowRequest.FromSerializedString(req)
      request.needs_processing = needs_processing
      request.nr_responses_expected = responses_expected
      request.timestamp = mysql_utils.TimestampToRDFDatetime(ts)
      requests[request.request_id] = request

    query = ("SELECT response, status, iterator, UNIX_TIMESTAMP(timestamp) "
             "FROM flow_responses "
             "WHERE client_id=%s AND flow_id=%s")
    cursor.execute(query, args)

    responses = {}
    for res, status, iterator, ts in cursor.fetchall():
      if status:
        response = rdf_flow_objects.FlowStatus.FromSerializedString(status)
      elif iterator:
        response = rdf_flow_objects.FlowIterator.FromSerializedString(iterator)
      else:
        response = rdf_flow_objects.FlowResponse.FromSerializedString(res)
      response.timestamp = mysql_utils.TimestampToRDFDatetime(ts)
      responses.setdefault(response.request_id, []).append(response)

    res = {}
    while next_needed_request in requests:
      req = requests[next_needed_request]
      sorted_responses = sorted(
          responses.get(next_needed_request, []), key=lambda r: r.response_id)
      res[req.request_id] = (req, sorted_responses)
      next_needed_request += 1

    return res
100891
def read(self, request):
        
        pipeline = view_pipeline_factory.read_pipeline(
            configuration=self.configuration)
        return pipeline(request=request)
252450
private void doRead() {
        if (context == null) {
            context = vertx.getOrCreateContext();
        }
        if (state == STATUS_ACTIVE) {
            final Handler<Buffer> dataHandler = this.dataHandler;
            final Handler<Void> closeHandler = this.closeHandler;
            executor.submit(
                    (Runnable) () -> {
                        try {
                            final byte[] bytes = readChunk();

                            if (bytes == null || bytes.length == 0) {
                                // null or 0 means we reach the end of the stream, invoke the close handler.
                                state = STATUS_CLOSED;
                                IOUtils.closeQuietly(in);
                                context.runOnContext(event -> {
                                    if (closeHandler != null) {
                                        closeHandler.handle(null);
                                    }
                                });
                            } else {
                                // We still have data, dispatch it.
                                context.runOnContext(event -> {
                                    dataHandler.handle(Buffer.buffer(bytes));
                                    // The next chunk will be read in another call, and maybe another thread.
                                    // As the data was already given to the data handler, this is fine.
                                    doRead();
                                });
                            }
                        } catch (final Exception e) {
                            // Error detected, invokes the failure handler.
                            state = STATUS_CLOSED;
                            IOUtils.closeQuietly(in);
                            /**
                             * Invokes the failure handler.
                             * @param event irrelevant
                             */
                            context.runOnContext(event -> {
                                if (failureHandler != null) {
                                    failureHandler.handle(e);
                                }
                            });
                        }
                    });
        }
    }
1083130
def release_read_lock
      while(true)
        c = @counter.value
        if @counter.compare_and_swap(c,c-1)
          if waiting_writer?(c) && running_readers(c) == 1
            @writer_mutex.synchronize { @writer_q.signal }
          end
          break
        end
      end
      true
    end
1047375
def _ProcessLockAcquired(self):
        """"""
        try:
            is_locked = self._process_lock.acquire(timeout=self._lock_timeout)
            yield is_locked
        finally:
            if is_locked:
                self._process_lock.release()
118724
private void adjustRequestProcessorsPool()
    {
        assertLocked();

        AttrRequests attrRequests = cmParams.getRequests();

        int requests
            = attrRequests != null
                ? attrRequests.intValue() : 2;

        // NOTE In polling mode with default WAIT=60 connection
        //      will be unresponsive
        if (requests <= 1 && "1".equals(String.valueOf(cmParams.getHold()))) {
            LOG.warning(
                "CM supports only 1 requests at a time and there is"
                    + " a risk of connection being stuck up to "
                    + cmParams.getWait() + "seconds");
        }

        // Expand request processors pool
        if (requests > procThreads.length) {

            RequestProcessor[] oldProcessors = procThreads;
            procThreads = new RequestProcessor[requests];

            System.arraycopy(
                oldProcessors, 0, procThreads, 0, oldProcessors.length);

            for (int i = oldProcessors.length; i < requests; i++) {
                procThreads[i] = new RequestProcessor(i);
                procThreads[i].start();
            }
        }
    }
322261
def _ProcessCompletedRequests(self, notification):
    """"""
    # First ensure that client messages are all removed. NOTE: We make a new
    # queue manager here because we want only the client messages to be removed
    # ASAP. This must happen before we actually run the flow to ensure the
    # client requests are removed from the client queues.
    with queue_manager.QueueManager(token=self.token) as manager:
      for request, _ in manager.FetchCompletedRequests(
          self.session_id, timestamp=(0, notification.timestamp)):
        # Requests which are not destined to clients have no embedded request
        # message.
        if request.HasField("request"):
          manager.DeQueueClientRequest(request.request)

    # The flow is dead - remove all outstanding requests and responses.
    if not self.IsRunning():
      self.queue_manager.DestroyFlowStates(self.session_id)
      return

    processing = []
    while True:
      try:
        # Here we only care about completed requests - i.e. those requests with
        # responses followed by a status message.
        for request, responses in self.queue_manager.FetchCompletedResponses(
            self.session_id, timestamp=(0, notification.timestamp)):

          if request.id == 0:
            continue

          if not responses:
            break

          # We are missing a needed request - maybe its not completed yet.
          if request.id > self.context.next_processed_request:
            stats_collector_instance.Get().IncrementCounter(
                "grr_response_out_of_order")
            break

          # Not the request we are looking for - we have seen it before
          # already.
          if request.id < self.context.next_processed_request:
            self.queue_manager.DeleteRequest(request)
            continue

          if not responses:
            continue

          # Do we have all the responses here? This can happen if some of the
          # responses were lost.
          if len(responses) != responses[-1].response_id:
            # If we can retransmit do so. Note, this is different from the
            # automatic retransmission facilitated by the task scheduler (the
            # Task.task_ttl field) which would happen regardless of these.
            if request.transmission_count < 5:
              stats_collector_instance.Get().IncrementCounter(
                  "grr_request_retransmission_count")
              request.transmission_count += 1
              self.ReQueueRequest(request)
            break

          # If we get here its all good - run the flow.
          if self.IsRunning():
            self.flow_obj.HeartBeat()
            self.RunStateMethod(request.next_state, request, responses)

          # Quit early if we are no longer alive.
          else:
            break

          # At this point we have processed this request - we can remove it and
          # its responses from the queue.
          self.queue_manager.DeleteRequest(request)
          self.context.next_processed_request += 1
          self.DecrementOutstandingRequests()

        # Are there any more outstanding requests?
        if not self.OutstandingRequests():
          # Allow the flow to cleanup
          if self.IsRunning() and self.context.current_state != "End":
            self.RunStateMethod("End")

        # Rechecking the OutstandingRequests allows the End state (which was
        # called above) to issue further client requests - hence postpone
        # termination.
        if not self.OutstandingRequests():
          # TODO(user): Deprecate in favor of 'flow_completions' metric.
          stats_collector_instance.Get().IncrementCounter(
              "grr_flow_completed_count")

          stats_collector_instance.Get().IncrementCounter(
              "flow_completions", fields=[self.flow_obj.Name()])
          logging.debug(
              "Destroying session %s(%s) for client %s", self.session_id,
              self.flow_obj.Name(), self.runner_args.client_id)

          self.flow_obj.Terminate()

        # We are done here.
        return

      except queue_manager.MoreDataException:
        # Join any threads.
        for event in processing:
          event.wait()

        # We did not read all the requests/responses in this run in order to
        # keep a low memory footprint and have to make another pass.
        self.FlushMessages()
        self.flow_obj.Flush()
        continue

      finally:
        # Join any threads.
        for event in processing:
          event.wait()
1207831
private void requestBlock() {
    next = ByteBuffer.allocate(blockSizeBytes);
    long requestTimeout = retryParams.getRequestTimeoutMillisForCurrentAttempt();
    pendingFetch = raw.readObjectAsync(next, filename, fetchPosition, requestTimeout);
  }
750882
protected void processRequest() {
        final int timeout = getHTTPContext().getReadTimeout();
        final TCPReadCompletedCallback callback = HttpICLReadCallback.getRef();

        // keep looping on processing information until we fully parse the message
        // and hand it off, or until the reads for more data go async
        VirtualConnection rc = null;
        do {
            if (handleNewInformation()) {
                // new information triggered an error message, so we're done
                return;
            }
            // Note: handleNewInformation will allocate the read buffers
            if (!isPartiallyParsed()) {
                // we're done parsing at this point. Note: cannot take any action
                // with information after this call because it may go all the way
                // from the channel above us back to the persist read, must exit
                // this callstack immediately
                handleNewRequest();
                return;
            }
            rc = this.myTSC.getReadInterface().read(1, callback, false, timeout);
        } while (null != rc);
    }
1117116
def next_requests(self):
        """"""
        use_set = self.settings.getbool('REDIS_START_URLS_AS_SET', defaults.START_URLS_AS_SET)
        fetch_one = self.server.spop if use_set else self.server.lpop
        # XXX: Do we need to use a timeout here?
        found = 0
        # TODO: Use redis pipeline execution.
        while found < self.redis_batch_size:
            data = fetch_one(self.redis_key)
            if not data:
                # Queue empty.
                break
            req = self.make_request_from_data(data)
            if req:
                yield req
                found += 1
            else:
                self.logger.debug("Request not made from data: %r", data)

        if found:
            self.logger.debug("Read %s requests from '%s'", found, self.redis_key)