Run processes from queue
163144
def _run_parallel_multiprocess(self):
        
        _log.debug("run.parallel.multiprocess.start")
        processes = []
        ProcRunner.instance = self
        for i in range(self._ncores):
            self._status.running(i)
            proc = multiprocessing.Process(target=ProcRunner.run, args=(i,))
            proc.start()
            processes.append(proc)
        for i in range(self._ncores):
            processes[i].join()
            code = processes[i].exitcode
            self._status.success(i) if 0 == code else self._status.fail(i)
        _log.debug("run.parallel.multiprocess.end states={}".format(self._status))
356591
function queue(args) {
    args.push(resultsCallback);
    stack.push({caller: this.caller, args: args});
  }
212350
def get_jobs_from_queue(self, queue: str, max_jobs: int) -> List[Job]:
        """"""
        jobs_json_string = self._run_script(
            self._get_jobs_from_queue,
            self._to_namespaced(queue),
            self._to_namespaced(RUNNING_JOBS_KEY.format(self._id)),
            JobStatus.RUNNING.value,
            max_jobs
        )

        jobs = json.loads(jobs_json_string.decode())
        jobs = [Job.deserialize(job) for job in jobs]

        return jobs
167181
def _run_process(self, start_path, stop_path, process_num=0):
        
        # pre processing
        self.producer.initialize_worker(process_num)
        self.consumer.initialize_worker(process_num)

        # processing
        for path in range(start_path, stop_path):
            self._run_path(path)

        # post processing
        self.consumer.finalize_worker(process_num)
1223988
def process_queues(self, forced=False):
        """"""

        with self.lock:
            if (not forced and not self.must_process) or not self.queues_enabled:
                return
            self.must_process = False
        ARBITRATOR.awaken()
412004
def multithreader(args, paths):
    """"""

    def shellprocess(path):
        """Return a ready-to-use subprocess."""
        import subprocess
        return subprocess.Popen(args + [path],
                                stderr=subprocess.DEVNULL,
                                stdout=subprocess.DEVNULL)

    processes = [shellprocess(path) for path in paths]

    for process in processes:
        process.wait()
682239
def run
      done = []

      queues.each_with_object([]) {|(fqdn, queue), workers|
        @thread_limit.times.map.with_index {|i|
          workers << Thread.start {
            Thread.current[:id] = i+1
            done.push *worker_class(fqdn).new(fqdn, queue(fqdn), cache(fqdn), &@block).run(@thread_limit)
          }
        }
      }.each {|t| t.join }

      done
    end
920906
def process_queue(queue=None, **kwargs):
    
    while True:
        item = queue.get()
        if item is None:
            queue.task_done()
            logger.info(f"{queue}: exiting process queue.")
            break
        filename = os.path.basename(item)
        try:
            queue.next_task(item, **kwargs)
        except Exception as e:
            queue.task_done()
            logger.warn(f"{queue}: item={filename}. {e}\n")
            logger.exception(e)
            sys.stdout.write(
                style.ERROR(
                    f"{queue}. item={filename}. {e}. Exception has been logged.\n"
                )
            )
            sys.stdout.flush()
            break
        else:
            logger.info(f"{queue}: Successfully processed {filename}.\n")
        queue.task_done()
301377
private function triggerQueueNow()
    {
        $url = UrlHelper::actionUrl('queue/run');

        if (\function_exists('curl_init')) {
            $ch = curl_init($url);

            $options = [
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_CONNECTTIMEOUT => false,
                CURLOPT_NOSIGNAL => true
            ];

            if (\defined('CURLOPT_TIMEOUT_MS')) {
                $options[CURLOPT_TIMEOUT_MS] = 500;
            } else {
                $options[CURLOPT_TIMEOUT] = 1;
            }

            curl_setopt_array($ch, $options);
            curl_exec($ch);
            curl_close($ch);
        }
    }
28168
def run_in_thread_pool(self, pool_size=None, func=None, *args):
        """"""
        executor = Pool(pool_size)
        return self.loop.run_in_executor(executor, func, *args)
427500
def _run_queued_callbacks(self):
        '''  '''
        for callback in self._queued_callbacks:
            try:
                callback()
            except Exception as ex:
                logger.error("Exception: %s" % str(ex))
410294
function runQueue(){
			if( urllist.length ){
				AI.makeReq( o.proxy + urllist.join( "," ), elQueue );
				elQueue = $();
				urllist = [];
			}
		}
261785
def runm():
    
    signal.signal(signal.SIGINT, signal_handler)
    count = int(sys.argv.pop(1))
    processes = [Process(target=run, args=()) for x in range(count)]
    try:
        for p in processes:
            p.start()
    except KeyError:
        # Not sure why we see a keyerror here. Weird.
        pass
    finally:
        for p in processes:
            p.join()
1371559
def start(self, timeout=None):
        

        if self.is_alive():
            log.warning("a process with pid %s is already running", self._proc.pid)
            return
            
        self._run.value = True
        self._func_running.value = False
        name = self.__class__.__name__
        
        self.conn_recv, self.conn_send = mp.Pipe(False)
        self._monitor_thread = threading.Thread(target = self._monitor_stdout_pipe)
        self._monitor_thread.daemon=True
        self._monitor_thread.start()
        log.debug("started monitor thread")
        self._log_queue = mp.Queue()
        self._log_queue_listener = QueueListener(self._log_queue, *log.handlers)
        self._log_queue_listener.start()

        args = (self.func, self.args, self._run, self._pause, self.interval,
                self._sigint, self._sigterm, name, log.level, self.conn_send, 
                self._func_running, self._log_queue)
        
        self._proc = mp.Process(target = _loop_wrapper_func,
                                args   = args)
        self._proc.start()
        log.info("started a new process with pid %s", self._proc.pid)
        log.debug("wait for loop function to come up")
        t0 = time.time()
        while not self._func_running.value:
            if self._proc.exitcode is not None:
                exc = self._proc.exitcode
                self._proc = None
                if exc == 0:
                    log.warning("wrapper function already terminated with exitcode 0\nloop is not running")
                    return
                else:
                    raise LoopExceptionError("the loop function return non zero exticode ({})!\n".format(exc)+
                                             "see log (INFO level) for traceback information")
            
            time.sleep(0.1)           
            if (timeout is not None) and ((time.time() - t0) > timeout):
                err_msg = "could not bring up function on time (timeout: {}s)".format(timeout)
                log.error(err_msg)
                log.info("either it takes too long to spawn the subprocess (increase the timeout)\n"+
                         "or an internal error occurred before reaching the function call")
                raise LoopTimeoutError(err_msg)
            
        log.debug("loop function is up ({})".format(humanize_time(time.time()-t0)))
1274077
def run_multiple_processes(args_list: List[List[str]],
                           die_on_failure: bool = True) -> None:
    
    for procargs in args_list:
        start_process(procargs)
    # Wait for them all to finish
    wait_for_processes(die_on_failure=die_on_failure)
265097
def _run_parallel_process(self, start_path, stop_path, queue):
        
        process_num = int(current_process().name.split('-', 2)[1])
        self._run_process(start_path, stop_path, process_num)
        queue.put(self.consumer.put())
277716
def run(self):
    """"""
    while not self._state.is_terminating:
      self._process_event_queue()
      self._state.maybe_pause()