Create values to be used by upload_folder_run function.
        :param message_queue: Queue: queue background process can send messages to us on
        :param task_id: int: id of this command's task so message will be routed correctly
773313
def create_context(self, message_queue, task_id):
        
        params = (self.remote_folder.name, self.parent.kind, self.parent.remote_id)
        return UploadContext(self.settings, params, message_queue, task_id)
348221
protected void sendMessageToInstance(int taskId, HeronTuples.HeronTupleSet message) {
    taskIdToInstanceExecutor.get(taskId).getStreamInQueue().offer(message);
  }
497220
def _process_queue(self):
        '''
        
        '''
        if len(self._queue):
            args, kwargs = self._queue.popleft()
            self.publish(*args, **kwargs)
103624
def create_context(self, message_queue, task_id):
        
        params = (self.settings.dest_directory, self.file_url.json_data, self.seek_amt, self.bytes_to_read)
        return DownloadContext(self.settings, params, message_queue, task_id)
1360114
function add(message) {
        if (!message.order || // no keep order needed
                message.inProcess) {// or already in process
            return false; // should continue
        }

        var queue = this.getQueue(message.context);
        queue.unshift(message); // FIFO
        message.pointId = this.pipe._id;
        var moreInQueue = queue.length > 1;
        message.inProcess = !moreInQueue;

        return moreInQueue;
    }
814417
def _heartbeat(self, queue, task_ids):
        
        now = time.time()
        self.connection.zadd(self._key(ACTIVE, queue),
                             **{task_id: now for task_id in task_ids})
191024
def _process_queue_tasks(self, queue, queue_lock, task_ids, now, log):
        """"""

        processed_count = 0

        # Get all tasks
        serialized_tasks = self.connection.mget([
            self._key('task', task_id) for task_id in task_ids
        ])

        # Parse tasks
        tasks = []
        for task_id, serialized_task in zip(task_ids, serialized_tasks):
            if serialized_task:
                task_data = json.loads(serialized_task)
            else:
                # In the rare case where we don't find the task which is
                # queued (see ReliabilityTestCase.test_task_disappears),
                # we log an error and remove the task below. We need to
                # at least initialize the Task object with an ID so we can
                # remove it.
                task_data = {'id': task_id}

            task = Task(self.tiger, queue=queue, _data=task_data,
                        _state=ACTIVE, _ts=now)

            if not serialized_task:
                # Remove task as per comment above
                log.error('not found', task_id=task_id)
                task._move()
            elif task.id != task_id:
                log.error('task ID mismatch', task_id=task_id)
                # Remove task
                task._move()
            else:
                tasks.append(task)

        # List of task IDs that exist and we will update the heartbeat on.
        valid_task_ids = set(task.id for task in tasks)

        # Group by task func
        tasks_by_func = OrderedDict()
        for task in tasks:
            func = task.serialized_func
            if func in tasks_by_func:
                tasks_by_func[func].append(task)
            else:
                tasks_by_func[func] = [task]

        # Execute tasks for each task func
        for tasks in tasks_by_func.values():
            success, processed_tasks = self._execute_task_group(queue,
                    tasks, valid_task_ids, queue_lock)
            processed_count = processed_count + len(processed_tasks)
            log.debug('processed', attempted=len(tasks),
                      processed=processed_count)
            for task in processed_tasks:
                self._finish_task_processing(queue, task, success)

        return processed_count
354697
def processpool_map(task, args, message, concurrency, batchsize=1, nargs=None):
    
    njobs = get_njobs(nargs, args)
    show_progress = bool(message)
    batches = grouper(batchsize, tupleise(args))
    def batched_task(*batch):
        return [task(*job) for job in batch]

    if show_progress:
        message += ' (PP:{}w:{}b)'.format(concurrency, batchsize)
        pbar = setup_progressbar(message, njobs, simple_progress=True)
        pbar.start()
    
    q_in   = multiprocessing.Queue()  # Should I limit either queue size? Limiting in-queue
    q_out  = multiprocessing.Queue()  # increases time taken to send jobs, makes pbar less useful

    proc = [multiprocessing.Process(target=fun, args=(batched_task, q_in, q_out)) for _ in range(concurrency)]
    for p in proc:
        p.daemon = True
        p.start()
    sent = [q_in.put((i, x)) for (i, x) in enumerate(batches)]
    [q_in.put((None, None)) for _ in range(concurrency)]
    res = []
    completed_count = 0
    for _ in range(len(sent)):
        result = get_from_queue(q_out)
        res.append(result)
        completed_count += len(result[1])
        if show_progress:
            pbar.update(completed_count)

    [p.join() for p in proc]
    if show_progress:
        pbar.finish()

    return flatten_list([x for (i, x) in sorted(res)])
792864
def _queue_task(self, args):
        '''  '''
        if self._cancel_called:
            raise AsperaTransferQueueError("Cancel already called")
        elif self._wait_called:
            raise AsperaTransferQueueError("Cant queue items during wait")
        elif self.waiting_coordinator_count() >= self._config.max_submission_queue_size:
            raise AsperaTransferQueueError("Max queued items reached")
        else:
            _coordinator = AsperaTransferCoordinator(args)
            _components = {'meta': TransferMeta(args, transfer_id=args.transfer_id),
                           'coordinator': _coordinator}

            _transfer_future = AsperaTransferFuture(**_components)
            _coordinator.add_subscribers(args.subscribers, future=_transfer_future)
            _coordinator.add_done_callback(self.remove_aspera_coordinator,
                                           transfer_coordinator=_coordinator)
            self.append_waiting_queue(_coordinator)

            if not self._processing_thread:
                self._processing_thread = threading.Thread(target=self._process_waiting_queue)
                self._processing_thread.daemon = True
                self._processing_thread.start()

            self._wakeup_processing_thread()

        return _transfer_future
801560
def exec_task(task_path, data):
    
    if not data:
        data = {'data': None, 'path': task_path}

    elif not isinstance(data, (str, bytes)):
        data = {'data': json.dumps(data, cls=RequestJSONEncoder),
                'path': task_path}

    else:
        # Open the data from file, if necessary.
        if data is not None and data.startswith("file://"):
            with open(data[len("file://"):]) as f:
                data = f.read()

        data = {'data': data, 'path': task_path}

    # Prepare the task.
    job = Job(data)
    (task, task_callable) = create_task(task_path)

    with delegating_job_context(job, task, task_callable) as jc:
        return jc.task_callable(jc.task_data)
91629
def submit(self, *args, func=None, monitor=None):
        
        monitor = monitor or self.monitor
        func = func or self.task_func
        if not hasattr(self, 'socket'):  # first time
            self.__class__.running_tasks = self.tasks
            self.socket = Socket(self.receiver, zmq.PULL, 'bind').__enter__()
            monitor.backurl = 'tcp://%s:%s' % (
                config.dbserver.host, self.socket.port)
        assert not isinstance(args[-1], Monitor)  # sanity check
        dist = 'no' if self.num_tasks == 1 else self.distribute
        if dist != 'no':
            args = pickle_sequence(args)
            self.sent += numpy.array([len(p) for p in args])
        res = submit[dist](self, func, args, monitor)
        self.tasks.append(res)
329157
def wait(self, task_id):
        
        data = {
            'action': 'wait',
            'id': task_id
        }
        self.__send_pyobj(data)
        task = self.__recv_pyobj(notimeout=True)
        return task
1099968
def process_messages_loop_internal(self):
        
        logging.info("Starting work queue loop.")
        self.connection.receive_loop_with_callback(self.queue_name, self.process_message)
88476
def insert(self):
        
        from google.appengine.api.taskqueue import Queue

        task = self.to_task()

        Queue(name=self.get_queue()).add(task)
652623
def add_message(self, msg_content, folder, **kwargs):
        
        content = {'m': kwargs}
        content['m']['l'] = str(folder)
        content['m']['content'] = {'_content': msg_content}

        return self.request('AddMsg', content)
136435
def insert(self, task, args=[], kwargs={}, delay_seconds=0):
    
    body = {
      "payload": task.payload(),
      "queueName": self._queue_name,
      "groupByTag": True,
      "tag": task.__class__.__name__
    }

    def cloud_insertion(api):
      api.insert(body, delay_seconds)

    if len(self._threads):
      self.put(cloud_insertion)
    else:
      cloud_insertion(self._api)

    return self
1272520
def task_response(self, message_id, data):
        
        try:
            response = None
            payload = json.dumps(data)

            logger.debug("Task response is %s: %s" % (self.__response_url(message_id), payload))

            response = self.client.post(self.__response_url(message_id),
                                        data=payload,
                                        headers={"Content-Type": "application/json"},
                                        timeout=0.8)
        except (requests.ConnectTimeout, requests.ConnectionError):
            logger.debug("task_response", exc_info=True)
        except Exception:
            logger.debug("task_response Exception", exc_info=True)
        finally:
            return response