Create *num_workers* worker objects with *input_tube* and 
        an iterable of *output_tubes*. The worker reads a task from *input_tube* 
        and writes the result to *output_tubes*.
344512
def init2(
        self, 
        input_tube,      # Read task from the input tube.
        output_tubes,    # Send result on all the output tubes.
        num_workers,     # Total number of workers in the stage.
        disable_result,  # Whether to override any result with None.
        do_stop_task,    # Whether to call doTask() on "stop" request.
        ):
        """"""

        super(UnorderedWorker, self).__init__()
        self._tube_task_input = input_tube
        self._tubes_result_output = output_tubes
        self._num_workers = num_workers
        self._disable_result = disable_result
        self._do_stop_task = do_stop_task
400388
def work():
    """"""
    global n

    cmd = utils.get_project_configuration()
    if 'worker_api_key' not in cmd:
        return ("You need to define a 'worker_api_key' in your ~/")

    chunk_size = 1000

    logging.info("Start working with n=%i", n)
    for _ in range(chunk_size):
        # contact the write-math server and get something to classify
        url = "http://www.martin-thoma.de/write-math/api/get_unclassified.php"
        response = urlopen(url)
        page_source = response.read()
        parsed_json = json.loads(page_source)
        if parsed_json is False:
            return "Nothing left to classify"
        raw_data_json = parsed_json['recording']

        # Classify
        # Check recording
        try:
            json.loads(raw_data_json)
        except ValueError:
            return ("Raw Data ID %s; Invalid JSON string: %s" %
                    (parsed_json['id'], raw_data_json))

        # Classify
        if use_segmenter_flag:
            strokelist = json.loads(raw_data_json)
            beam = se.Beam()
            for stroke in strokelist:
                beam.add_stroke(stroke)
            results = beam.get_writemath_results()
        else:
            results_sym = classify.classify_segmented_recording(raw_data_json)
            results = []
            strokelist = json.loads(raw_data_json)
            segmentation = [list(range(len(strokelist)))]
            translate = _get_translate()
            for symbol in results_sym:
                s = {'id': get_writemath_id(symbol, translate),
                     'probability': symbol['probability']}
                results.append({'probability': symbol['probability'],
                                'segmentation': segmentation,
                                'symbols': [s]})

        print("\thttp://write-math.com/view/?raw_data_id=%s" %
              str(parsed_json['id']))

        # Submit classification to write-math.com server
        results_json = get_json_result(results, n=n)
        headers = {'User-Agent': 'Mozilla/5.0',
                   'Content-Type': 'application/x-www-form-urlencoded'}
        payload = {'recording_id': parsed_json['id'],
                   'results': results_json,
                   'api_key': cmd['worker_api_key']}

        s = requests.Session()
        req = requests.Request('POST', url, headers=headers, data=payload)
        prepared = req.prepare()
        response = s.send(prepared)
        try:
            response = json.loads(response.text)
        except ValueError:
            return "Invalid JSON response: %s" % response.text

        if 'error' in response:
            logging.info(response)
            return str(response)
    return "Done - Classified %i recordings" % chunk_size
675642
def build_from_job_list(scheme_files, templates, base_output_dir):
    """"""
    queue = Queue()
    for scheme in scheme_files:
        queue.put(scheme)

    if len(scheme_files) < 40:
        thread_num = len(scheme_files)
    else:
        thread_num = 40

    threads = []
    for _ in range(thread_num):
        thread = Thread(target=build_single_worker,
                        args=(queue, templates, base_output_dir))
        thread.start()
        threads.append(thread)

    queue.join()

    for _ in range(thread_num):
        queue.put(None)

    for thread in threads:
        thread.join()
723259
def matrix_worker(data):
    
    matrix = data['matrix']
    Logger.get_logger(__name__ + '.worker').info(
        "Processing pipeline for matrix entry '%s'", matrix['name'])

    env = matrix['env'].copy()
    env.update({'PIPELINE_MATRIX': matrix['name']})

    pipeline = Pipeline(model=data['model'], env=env, options=data['options'])
    pipeline.hooks = data['hooks']
    return pipeline.process(data['pipeline'])
200057
public function create(): Worker
    {
        if (Parallel::isSupported()) {
            return new WorkerParallel($this->className);
        }

        if (Thread::isSupported()) {
            return new WorkerThread($this->className);
        }

        return new WorkerProcess(
            $this->className,
            [],
            \getenv("AMP_PHP_BINARY") ?: (\defined("AMP_PHP_BINARY") ? \AMP_PHP_BINARY : null)
        );
    }
977859
def foreach_worker(self, fn):
        
        results = ray.get([w.foreach_worker.remote(fn) for w in self.workers])
        return results
81729
def outputscreate():
    
    # Create table with a nested index
    r.db("Brain").table_create("Outputs").run()
    r.db("Brain").table("Outputs").index_create("Output_job_id", r.row["OutputJob"]["id"]).run()
    r.db("Brain").table("Outputs").index_wait("Output_job_id").run()
350781
def _reliure_worker(wnum, Qin, Qout, pipeline, options={}):
    
    #pipeline = get_pipeline()
    logger = logging.getLogger("reliure.run_parallel.worker#%s" % wnum)
    logger.debug("worker created")
    if options is None:
        options = {}
    while True:
        chunk = Qin.get() # get an element (and wait for it if needed)
        logger.debug("Get %s elements to process" % len(chunk))
        res = [output for output in pipeline(chunk, **options)]
        logger.debug("processing done, results len = %s" % len(res))
        Qout.put(res)
        Qin.task_done()
946098
def initialize_worker(self, process_num=None):
        
        for p in self.producers:
            p.initialize_worker(process_num)
        # self.initial_state.process = process_num
        self.random.seed(hash(self.seed) + hash(process_num))
970038
def make_jobs():
    

    # set up list of arguments
    inputvec = [[3], [5], [10], [20]]

    # create empty job vector
    jobs = []

    # create job objects
    for arg in inputvec:
        # The default queue used by the Job class is all.q. You must specify
        # the `queue` keyword argument if that is not the name of your queue.
        job = Job(compute_factorial, arg, queue='all.q')
        jobs.append(job)

    return jobs
282257
def write(self, learn:Learner, trn_batch:Tuple, val_batch:Tuple, iteration:int, tbwriter:SummaryWriter)->None:
        ""
        self._write_for_dstype(learn=learn, batch=val_batch, iteration=iteration, tbwriter=tbwriter, ds_type=DatasetType.Valid)
        self._write_for_dstype(learn=learn, batch=trn_batch, iteration=iteration, tbwriter=tbwriter, ds_type=DatasetType.Train)
414637
def apply(self, func, workers=1, job_size=10000):
    
    if workers == 1:
      for lines in self.iter_chunks(job_size):
        yield func(lines)
    else:
      with ProcessPoolExecutor(max_workers=workers) as executor:
        for result in executor.map(func, self.iter_chunks(job_size)):
          yield result
209963
public static Set<Integer> worker_output_tasks(WorkerData workerData) {
        ContextMaker context_maker = workerData.getContextMaker();
        Set<Integer> taskIds = workerData.getTaskIds();
        StormTopology topology = workerData.getSysTopology();

        Set<Integer> rtn = new HashSet<>();

        for (Integer taskId : taskIds) {
            TopologyContext context = context_maker.makeTopologyContext(topology, taskId, null);

            // <StreamId, <ComponentId, Grouping>>
            Map<String, Map<String, Grouping>> targets = context.getThisTargets();
            for (Map<String, Grouping> e : targets.values()) {
                for (String componentId : e.keySet()) {
                    List<Integer> tasks = context.getComponentTasks(componentId);
                    rtn.addAll(tasks);
                }
            }
        }

        return rtn;
    }
1246104
def batch(input_iter, batch_size=32):
  """"""
  input_iter = iter(input_iter)
  next_ = list(itertools.islice(input_iter, batch_size))
  while next_:
    yield next_
    next_ = list(itertools.islice(input_iter, batch_size))
583510
def worker(self, fifo):
        '''
        '''
        logging.debug('Starting worker thread for %s', fifo)
        self._fifo_conditions[fifo].acquire()
        while True:
            try:
                data_tuple = self._fifo_data_deque[fifo].popleft()
            except IndexError:
                self._fifo_conditions[fifo].wait(self.readout_interval)  # sleep a little bit, reducing CPU usage
            else:
                if data_tuple is None:  # if None then exit
                    break
                else:
                    for index, (filter_func, converter_func, fifo_select) in enumerate(izip(self.filter_func, self.converter_func, self.fifo_select)):
                        if fifo_select is None or fifo_select == fifo:
                            # filter and do the conversion
                            converted_data_tuple = convert_data_iterable((data_tuple,), filter_func=filter_func, converter_func=converter_func)[0]
                            n_data_words = converted_data_tuple[0].shape[0]
                            with self.data_words_per_second_lock:
                                self._words_per_read[index].append((n_data_words, converted_data_tuple[1], converted_data_tuple[2]))
                            self._data_deque[index].append(converted_data_tuple)
                            with self._data_conditions[index]:
                                self._data_conditions[index].notify_all()
        for index, fifo_select in enumerate(self.fifo_select):
            if fifo_select is None or fifo_select == fifo:
                self._data_deque[index].append(None)
                with self._data_conditions[index]:
                    self._data_conditions[index].notify_all()
        self._fifo_conditions[fifo].release()
        logging.debug('Stopping worker thread for %s', fifo)
523804
def _thread_worker_fn(samples, batchify_fn, dataset):
    """"""
    if isinstance(samples[0], (list, tuple)):
        batch = [batchify_fn([dataset[i] for i in shard]) for shard in samples]
    else:
        batch = batchify_fn([dataset[i] for i in samples])
    return batch
1088534
def create_tasks(self, wfk_file, scr_input):
        
        assert len(self) == 0
        wfk_file = self.wfk_file = os.path.abspath(wfk_file)

        # Build a temporary work in the tmpdir that will use a shell manager
        # to run ABINIT in order to get the list of q-points for the screening.
        shell_manager = self.manager.to_shell_manager(mpi_procs=1)

        w = Work(workdir=self.tmpdir.path_join("_qptdm_run"), manager=shell_manager)

        fake_input = scr_input.deepcopy()
        fake_task = w.register(fake_input)
        w.allocate()
        w.build()

        # Create the symbolic link and add the magic value
        # nqpdm = -1 to the input to get the list of q-points.
        fake_task.inlink_file(wfk_file)
        fake_task.set_vars({"nqptdm": -1})
        fake_task.start_and_wait()

        # Parse the section with the q-points
        with NetcdfReader(fake_task.outdir.has_abiext("qptdms.nc")) as reader:
            qpoints = reader.read_value("reduced_coordinates_of_kpoints")
        #print("qpoints)

        # Now we can register the task for the different q-points
        for qpoint in qpoints:
            qptdm_input = scr_input.deepcopy()
            qptdm_input.set_vars(nqptdm=1, qptdm=qpoint)
            new_task = self.register_scr_task(qptdm_input, manager=self.manager)
            # Add the garbage collector.
            if self.flow.gc is not None:
                new_task.set_gc(self.flow.gc)

        self.allocate()