Create *num_workers* worker objects with *input_tube* and 
        an iterable of *output_tubes*. The worker reads a task from *input_tube* 
        and writes the result to *output_tubes*.
347672
def init2(
        self, 
        input_tube,      # Read task from the input tube.
        output_tubes,    # Send result on all the output tubes.
        num_workers,     # Total number of workers in the stage.
        disable_result,  # Whether to override any result with None.
        do_stop_task,    # Whether to call doTask() on "stop" request.
        ):
        """"""

        super(OrderedWorker, self).__init__()
        self._tube_task_input = input_tube
        self._tubes_result_output = output_tubes
        self._num_workers = num_workers

        # Serializes reading from input tube.
        self._lock_prev_input = None
        self._lock_next_input = None

        # Serializes writing to output tube.
        self._lock_prev_output = None
        self._lock_next_output = None

        self._disable_result = disable_result
        self._do_stop_task = do_stop_task
1222481
def _define_output_buffers(self):
        

        # First define buffers that need input data
        self.target_buffers = {
            None: [(step, self.buffers[step]) for step in self._get_input_steps()]
        }

        # Go through all steps and append the buffers of their child nodes
        for step in self.steps_sorted:
            if step != self:
                child_steps = [edge[1] for edge in self.graph.out_edges(step)]
                self.target_buffers[step] = [(child_step, self.buffers[child_step]) for child_step in child_steps]
723259
def matrix_worker(data):
    
    matrix = data['matrix']
    Logger.get_logger(__name__ + '.worker').info(
        "Processing pipeline for matrix entry '%s'", matrix['name'])

    env = matrix['env'].copy()
    env.update({'PIPELINE_MATRIX': matrix['name']})

    pipeline = Pipeline(model=data['model'], env=env, options=data['options'])
    pipeline.hooks = data['hooks']
    return pipeline.process(data['pipeline'])
844291
def _worker_fn(samples, batchify_fn, dataset=None):
    """"""
    # pylint: disable=unused-argument
    # it is required that each worker process has to fork a new MXIndexedRecordIO handle
    # preserving dataset as global variable can save tons of overhead and is safe in new process
    global _worker_dataset
    if isinstance(samples[0], (list, tuple)):
        batch = [batchify_fn([_worker_dataset[i] for i in shard]) for shard in samples]
    else:
        batch = batchify_fn([_worker_dataset[i] for i in samples])
    buf = io.BytesIO()
    ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(batch)
    return buf.getvalue()
1246104
def batch(input_iter, batch_size=32):
  """"""
  input_iter = iter(input_iter)
  next_ = list(itertools.islice(input_iter, batch_size))
  while next_:
    yield next_
    next_ = list(itertools.islice(input_iter, batch_size))
437800
def runMultiprocessNoiseExperiment(resultName=None, numWorkers = 0, **kwargs):
  

  if resultName is None:
    resultName = str(kwargs) + ".json"

  experiments = [{}]
  for key, values in kwargs.items():
    if type(values) is list:
      newExperiments = []
      for experiment in experiments:
        for val in values:
          newExperiment = copy(experiment)
          newExperiment[key] = val
          newExperiments.append(newExperiment)
      experiments = newExperiments
    else:
      [a.__setitem__(key, values) for a in experiments]

  if numWorkers == 0:
    numWorkers = cpu_count()
  if numWorkers > 1:
    pool = Pool(processes=numWorkers)
    rs = pool.map_async(experimentWrapper, experiments, chunksize=1)
    while not rs.ready():
      remaining = rs._number_left
      pctDone = 100.0 - (100.0*remaining) / len(experiments)
      print "    => {} experiments remaining, percent complete={}".format(\
        remaining, pctDone)
      time.sleep(5)
    pool.close()  # No more work
    pool.join()
    result = rs.get()
  else:
    result = []
    for arg in experiments:
      result.append(doExperiment(**arg))

  # Save results for later use
  results = [(arg,res) for arg, res in zip(experiments, result)]
  with open(resultName,"wb") as f:
    json.dump(results,f)

  return results
111218
def _start_workers(self):
        
        # creating the pool of worker process or threads
        log.debug('%s starts a %s-pool of %s workers.' % \
                  (self, self.worker_type, self.worker_num))
        self.pool = []
        for host, worker_num in \
                           [(None, self.worker_num)] + list(self.worker_remote):
            for _worker in range(worker_num):
                __worker = Thread(target=_pool_worker, args=\
                                  (self._inqueue, self._outqueue, host)) \
                                  if self.worker_type == 'thread' else \
                           Process(target=_pool_worker, args=\
                                  (self._inqueue, self._outqueue, host))
                self.pool.append(__worker)
        for __worker in self.pool:
            __worker.daemon = True
            __worker.start()
        log.debug('%s started the pool' % self)
675642
def build_from_job_list(scheme_files, templates, base_output_dir):
    """"""
    queue = Queue()
    for scheme in scheme_files:
        queue.put(scheme)

    if len(scheme_files) < 40:
        thread_num = len(scheme_files)
    else:
        thread_num = 40

    threads = []
    for _ in range(thread_num):
        thread = Thread(target=build_single_worker,
                        args=(queue, templates, base_output_dir))
        thread.start()
        threads.append(thread)

    queue.join()

    for _ in range(thread_num):
        queue.put(None)

    for thread in threads:
        thread.join()
946098
def initialize_worker(self, process_num=None):
        
        for p in self.producers:
            p.initialize_worker(process_num)
        # self.initial_state.process = process_num
        self.random.seed(hash(self.seed) + hash(process_num))
1183097
def new_job(self, task, inputdata, launcher_name="Unknown", debug=False):
        
        job_semaphore = threading.Semaphore(0)

        def manage_output(result, grade, problems, tests, custom, state, archive, stdout, stderr):
            """ Manages the output of this job """
            manage_output.job_return = (result, grade, problems, tests, custom, state, archive, stdout, stderr)
            job_semaphore.release()

        manage_output.job_return = None

        self._client.new_job(task, inputdata, manage_output, launcher_name, debug)
        job_semaphore.acquire()
        job_return = manage_output.job_return
        return job_return
3432
def workers():
    """"""
    counter = Counter()
    for w in Worker.all(connection=worker.connection):
        for q in w.queues:
            counter[q.name] += 1
    import pprint
    pprint.pprint(dict(counter))
1116304
def initialize_worker(self, process_num=None):
        
        self.initial_state.process = process_num
        self.random.seed(hash(self.seed) + hash(process_num))
970038
def make_jobs():
    

    # set up list of arguments
    inputvec = [[3], [5], [10], [20]]

    # create empty job vector
    jobs = []

    # create job objects
    for arg in inputvec:
        # The default queue used by the Job class is all.q. You must specify
        # the `queue` keyword argument if that is not the name of your queue.
        job = Job(compute_factorial, arg, queue='all.q')
        jobs.append(job)

    return jobs
209963
public static Set<Integer> worker_output_tasks(WorkerData workerData) {
        ContextMaker context_maker = workerData.getContextMaker();
        Set<Integer> taskIds = workerData.getTaskIds();
        StormTopology topology = workerData.getSysTopology();

        Set<Integer> rtn = new HashSet<>();

        for (Integer taskId : taskIds) {
            TopologyContext context = context_maker.makeTopologyContext(topology, taskId, null);

            // <StreamId, <ComponentId, Grouping>>
            Map<String, Map<String, Grouping>> targets = context.getThisTargets();
            for (Map<String, Grouping> e : targets.values()) {
                for (String componentId : e.keySet()) {
                    List<Integer> tasks = context.getComponentTasks(componentId);
                    rtn.addAll(tasks);
                }
            }
        }

        return rtn;
    }
919284
def worker(self):
        
        tasks_completed = 0
        for task in iter(self.task_queue.get, 'STOP'):
            col_index, item, item2 = task
            if not hasattr(item, '__iter__') or isinstance(item, tuple):
                item = [item]
            if not hasattr(item2, '__iter__') or isinstance(item2, tuple):
                item2 = [item2]
            result = (col_index, self.combinfunc(item, item2))
            self.done_queue.put(result)
            tasks_completed += 1
        logger.info("Worker %s performed %s tasks",
                    current_process().name,
                    tasks_completed)
1216815
def init(self, aggregators):
        """"""

        assert len(aggregators) == self.num_aggregation_workers, aggregators
        if len(self.remote_evaluators) < self.num_aggregation_workers:
            raise ValueError(
                "The number of aggregation workers should not exceed the "
                "number of total evaluation workers ({} vs {})".format(
                    self.num_aggregation_workers, len(self.remote_evaluators)))

        assigned_evaluators = collections.defaultdict(list)
        for i, ev in enumerate(self.remote_evaluators):
            assigned_evaluators[i % self.num_aggregation_workers].append(ev)

        self.workers = aggregators
        for i, worker in enumerate(self.workers):
            worker.init.remote(
                self.broadcasted_weights, assigned_evaluators[i],
                self.max_sample_requests_in_flight_per_worker,
                self.replay_proportion, self.replay_buffer_num_slots,
                self.train_batch_size, self.sample_batch_size)

        self.agg_tasks = TaskPool()
        for agg in self.workers:
            agg.set_weights.remote(self.broadcasted_weights)
            self.agg_tasks.add(agg, agg.get_train_batches.remote())

        self.initialized = True
894179
def create_python_worker(self, func, *args, **kwargs):
        """"""
        worker = PythonWorker(func, args, kwargs)
        self._create_worker(worker)
        return worker