Create a pull stream that receives jobs and passes them on to the
    database. If the job already has a result, that result is pushed onto
    the `result_queue`.
347313
def pass_job(db: JobDB, result_queue: Queue, always_cache=False):
    
    @pull
    def pass_job_stream(job_source):
        """Pull stream instance created by `pass_job`."""
        result_sink = result_queue.sink()

        for message in job_source():
            if message is EndOfQueue:
                return

            key, job = message
            if always_cache or ('store' in job.hints):
                status, retrieved_result = db.add_job_to_db(key, job)

                if status == 'retrieved':
                    result_sink.send(retrieved_result)
                    continue

                elif status == 'attached':
                    continue

            yield message

    return pass_job_stream
666294
def send_result(self, type, task, result):
        ''''''
        if self.outqueue:
            try:
                self.outqueue.put((task, result))
            except Exception as e:
                logger.exception(e)
705820
def job(func_or_queue=None, connection=None, *args, **kwargs):
    
    if callable(func_or_queue):
        func = func_or_queue
        queue = 'default'
    else:
        func = None
        queue = func_or_queue or 'default'

    if not isinstance(queue, basestring):
        queue = unicode(queue)

    try:
        queue = get_queue(queue)
        if connection is None:
            connection = queue.connection
    except KeyError:
        pass

    decorator = _job(queue, connection=connection, *args, **kwargs)
    if func:
        return decorator(func)
    return decorator
604127
def _job_done_callback(self, submissionid, task, result, grade, problems, tests, custom, state, archive, stdout, stderr, newsub=True):
        
        submission = self.get_submission(submissionid, False)
        submission = self.get_input_from_submission(submission)

        data = {
            "status": ("done" if result[0] == "success" or result[0] == "failed" else "error"),
             # error only if error was made by INGInious
            "result": result[0],
            "grade": grade,
            "text": result[1],
            "tests": tests,
            "problems": problems,
            "archive": (self._gridfs.put(archive) if archive is not None else None),
            "custom": custom,
            "state": state,
            "stdout": stdout,
            "stderr": stderr
        }

        unset_obj = {
            "jobid": "",
            "ssh_host": "",
            "ssh_port": "",
            "ssh_password": ""
        }

        # Save submission to database
        submission = self._database.submissions.find_one_and_update(
            {"_id": submission["_id"]},
            {"$set": data, "$unset": unset_obj},
            return_document=ReturnDocument.AFTER
        )

        self._hook_manager.call_hook("submission_done", submission=submission, archive=archive, newsub=newsub)

        for username in submission["username"]:
            self._user_manager.update_user_stats(username, task, submission, result[0], grade, state, newsub)

        if "outcome_service_url" in submission and "outcome_result_id" in submission and "outcome_consumer_key" in submission:
            for username in submission["username"]:
                self._lti_outcome_manager.add(username,
                                              submission["courseid"],
                                              submission["taskid"],
                                              submission["outcome_consumer_key"],
                                              submission["outcome_service_url"],
                                              submission["outcome_result_id"])
841174
def finish(self, result, array_id = None):
    """"""
    # check if there is any array job still running
    new_status = 'success' if result == 0 else 'failure'
    new_result = result
    finished = True
    if array_id is not None:
      for array_job in self.array:
        if array_job.id == array_id:
          array_job.status = new_status
          array_job.result = result
          array_job.finish_time = datetime.now()
        if array_job.status not in ('success', 'failure'):
          finished = False
        elif new_result == 0:
          new_result = array_job.result

    if finished:
      # There was no array job, or all array jobs finished
      self.status = 'success' if new_result == 0 else 'failure'
      self.result = new_result
      self.finish_time = datetime.now()

      # update all waiting jobs
      for job in self.get_jobs_waiting_for_us():
        if job.status == 'waiting':
          job.queue()
722929
def _get_results(FetcherClass, services, kwargs, num_results=None, fast=0, verbose=False, timeout=None):
    
    results = []

    if not num_results or fast:
        num_results = len(services)

    with futures.ThreadPoolExecutor(max_workers=len(services)) as executor:
        fetches = {}
        for service in services[:num_results]:
            tail = [x for x in services if x is not service]
            random.shuffle(tail)
            srv = FetcherClass(services=[service] + tail, verbose=verbose, timeout=timeout)
            fetches[executor.submit(srv.action, **kwargs)] = srv

        if fast == 1:
            raise NotImplementedError
            # ths code is a work in progress. futures.FIRST_COMPLETED works differently than I thought...
            to_iterate, still_going = futures.wait(fetches, return_when=futures.FIRST_COMPLETED)
            for x in still_going:
                try:
                    x.result(timeout=1.001)
                except futures._base.TimeoutError:
                    pass

        elif fast > 1:
            raise Exception("fast level greater than 1 not yet implemented")
        else:
            to_iterate = futures.as_completed(fetches)

        for future in to_iterate:
            service = fetches[future]
            results.append([service, future.result()])

    return results
1246342
def post(self, result_id, project_id):
        """"""

        result = db.session.query(Result).filter_by(id=result_id).first()

        if result is None:
            return jsonify({
                'result': None,
                'message': 'No interface defined for URL.'
            }), 404

        job_status = CommandsState.job_status(result.path_name)
        if job_status != JobStatus.RUNNING:
            if job_status == JobStatus.NO_EXTENSION_ERROR:
                return jsonify({
                    'message': '\'CommandsExtension\' is not set or disabled.'
                }), 400
            elif job_status == JobStatus.INITIALIZED:
                return jsonify({
                    'message': 'The target training job has not run, yet'
                }), 400
            elif job_status == JobStatus.STOPPED:
                return jsonify({
                    'message': 'The target training job has already stopped'
                }), 400
            else:
                return jsonify({
                    'message': 'Cannot get the target training job status'
                }), 400

        request_json = request.get_json()
        if request_json is None:
            return jsonify({
                'message': 'Empty request.'
            }), 400

        command_name = request_json.get('name', None)
        if command_name is None:
            return jsonify({
                'message': 'Name is required.'
            }), 400

        schedule = request_json.get('schedule', None)
        if not CommandItem.is_valid_schedule(schedule):
            return jsonify({
                'message': 'Schedule is invalid.'
            }), 400

        command = CommandItem(
            name=command_name,
        )

        command.set_request(
            CommandItem.REQUEST_OPEN,
            request_json.get('body', None),
            request_json.get('schedule', None)
        )

        commands = CommandItem.load_commands(result.path_name)
        commands.append(command)

        CommandItem.dump_commands(commands, result.path_name)

        new_result = crawl_result(result, force=True)
        new_result_dict = new_result.serialize

        return jsonify({'commands': new_result_dict['commands']})
805205
function socketPull(jobObject, stdoutStreamOverride, stderrStreamOverride) {
    if (stdoutStreamOverride)
        logger.debug("Shimmering Socket job pulling");
    else
        logger.debug("Genuine socket job pulling");
    //  logger.debug(`${util.format(stdout)}`);
    let stdoutStream = stdoutStreamOverride ? stdoutStreamOverride : jobObject.stdout();
    let stderrStream = stderrStreamOverride ? stderrStreamOverride : jobObject.stderr();
    ss(jobObject.socket).on(`${jobObject.id}:stdout`, function (stream) {
        stdoutStream.then((_stdout) => {
            logger.debug(`Pumping stdout [${jobObject.id}:stdout]`);
            //logger.warn(`stdoutStream expected ${util.format(_stdout)}`);
            _stdout.pipe(stream);
        });
    });
    ss(jobObject.socket).on(`${jobObject.id}:stderr`, function (stream) {
        stderrStream.then((_stderr) => {
            logger.debug(`Pumping stderr [${jobObject.id}:stderr]`);
            //logger.warn(`stderrStream expected ${util.format(_stderr)}`);
            _stderr.pipe(stream);
        });
    });
    jobObject.socket.emit('completed', JSON.stringify(jobObject));
}
52002
def join(self):
        
        self._jobs.join()
        try:
            return self._results, self._errors
        finally:
            self._clear()
299852
def run_to_queue(self, queue, conn, options=None):
    """"""
    if options is None:
      # Default options.
      offset = None
      limit = None
      keys_only = None
    else:
      # Capture options we need to simulate.
      offset = options.offset
      limit = options.limit
      keys_only = options.keys_only

      # Cursors are supported for certain orders only.
      if (options.start_cursor or options.end_cursor or
          options.produce_cursors):
        names = set()
        if self.__orders is not None:
          names = self.__orders._get_prop_names()
        if '__key__' not in names:
          raise datastore_errors.BadArgumentError(
              '_MultiQuery with cursors requires __key__ order')

    # Decide if we need to modify the options passed to subqueries.
    # NOTE: It would seem we can sometimes let Cloud Datastore handle
    # the offset natively, but this would thwart the duplicate key
    # detection, so we always have to emulate the offset here.
    # We can set the limit we pass along to offset + limit though,
    # since that is the maximum number of results from a single
    # subquery we will ever have to consider.
    modifiers = {}
    if offset:
      modifiers['offset'] = None
      if limit is not None:
        modifiers['limit'] = min(_MAX_LIMIT, offset + limit)
    if keys_only and self.__orders is not None:
      modifiers['keys_only'] = None
    if modifiers:
      options = QueryOptions(config=options, **modifiers)

    if offset is None:
      offset = 0

    if limit is None:
      limit = _MAX_LIMIT

    if self.__orders is None:
      # Run the subqueries sequentially; there is no order to keep.
      keys_seen = set()
      for subq in self.__subqueries:
        if limit <= 0:
          break
        subit = tasklets.SerialQueueFuture('_MultiQuery.run_to_queue[ser]')
        subq.run_to_queue(subit, conn, options=options)
        while limit > 0:
          try:
            batch, index, result = yield subit.getq()
          except EOFError:
            break
          if keys_only:
            key = result
          else:
            key = result._key
          if key not in keys_seen:
            keys_seen.add(key)
            if offset > 0:
              offset -= 1
            else:
              limit -= 1
              queue.putq((None, None, result))
      queue.complete()
      return

    # This with-statement causes the adapter to set _orig_pb on all
    # entities it converts from protobuf.
    # TODO: Does this interact properly with the cache?
    with conn.adapter:
      # Start running all the sub-queries.
      todo = []  # List of (subit, dsquery) tuples.
      for subq in self.__subqueries:
        dsquery = subq._get_query(conn)
        subit = tasklets.SerialQueueFuture('_MultiQuery.run_to_queue[par]')
        subq.run_to_queue(subit, conn, options=options, dsquery=dsquery)
        todo.append((subit, dsquery))

      # Create a list of (first-entity, subquery-iterator) tuples.
      state = []  # List of _SubQueryIteratorState instances.
      for subit, dsquery in todo:
        try:
          thing = yield subit.getq()
        except EOFError:
          continue
        else:
          state.append(_SubQueryIteratorState(thing, subit, dsquery,
                                              self.__orders))

      # Now turn it into a sorted heap.  The heapq module claims that
      # calling heapify() is more efficient than calling heappush() for
      # each item.
      heapq.heapify(state)

      # Repeatedly yield the lowest entity from the state vector,
      # filtering duplicates.  This is essentially a multi-way merge
      # sort.  One would think it should be possible to filter
      # duplicates simply by dropping other entities already in the
      # state vector that are equal to the lowest entity, but because of
      # the weird sorting of repeated properties, we have to explicitly
      # keep a set of all keys, so we can remove later occurrences.
      # Note that entities will still be sorted correctly, within the
      # constraints given by the sort order.
      keys_seen = set()
      while state and limit > 0:
        item = heapq.heappop(state)
        batch = item.batch
        index = item.index
        entity = item.entity
        key = entity._key
        if key not in keys_seen:
          keys_seen.add(key)
          if offset > 0:
            offset -= 1
          else:
            limit -= 1
            if keys_only:
              queue.putq((batch, index, key))
            else:
              queue.putq((batch, index, entity))
        subit = item.iterator
        try:
          batch, index, entity = yield subit.getq()
        except EOFError:
          pass
        else:
          item.batch = batch
          item.index = index
          item.entity = entity
          heapq.heappush(state, item)
      queue.complete()
761858
def execute_task(self, task, parent_result):
        
        task.before_run(parent_result)
        context = task.create_context(self.message_queue)
        pending_result = self.pool.apply_async(execute_task_async, (task.func, task.id, context))
        self.pending_results.append(pending_result)
54377
def _queue_job(redis, queue_name, delegate, *args):
        
        log.info('Queuing job...')
        with Connection(redis):
            q = Queue(queue_name)
            job = q.enqueue(delegate, *args)
            return job.id
266269
def retrieve_job(self, job_id):
        
        warnings.warn('Use results generator method instead of retrieve_job which returns a list', UserWarning)
        return list(self.results(job_id))
465671
function queue (job, value, put) {
      var ts = timestamp()
      var key = toKey(job, ts)

      var id = hash(job+':'+value)
      if(pending[id]) return null //this job is already queued.
      pending[id] = Date.now()

      if(put === false) {
        //return the job to be queued, to include it in a batch insert.
          return {
          type: 'put', 
          key: Buffer.isBuffer(key) ? key : new Buffer(key), 
          value: Buffer.isBuffer(key) ? value : new Buffer(value)
        }
      } else {
        db.put(key, value)
      }
    }
658547
def job_success(self, job, queue, job_result):
        
        job.queued.delete()
        job.hmset(end=str(datetime.utcnow()), status=STATUSES.SUCCESS)
        queue.success.rpush(job.ident)
        self.log(self.job_success_message(job, queue, job_result))
        if hasattr(job, 'on_success'):
            job.on_success(queue, job_result)
673874
def execute(self, requests, resp_generator, *args, **kwargs):
        '''
            
        '''
        result_futures = [self.executor_pool.submit(resp_generator, req, *args, **kwargs) for req in requests]
        resp = [res_future.result() for res_future in result_futures]
        return resp
565877
public function push($job, array $data = null, $queue = null)
    {
        if (false !== ($delay = \Resque::getConfig('default.jobs.delay', false))) {
            return $this->later($delay, $job, $data, $queue);
        }

        return Job::create($this->getQueue($queue), $job, $data);
    }