Helper for multiprocessing.
166051
def __RemoteExecuteHelper(args):
  
  cmd, hostname, ssh_key = args
  #Random.atfork()  # needed to fix bug in old python 2.6 interpreters
  private_key = paramiko.RSAKey.from_private_key(StringIO.StringIO(ssh_key))
  client = paramiko.SSHClient()
  client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  while True:
      try:
          client.connect(hostname, username='ubuntu', pkey=private_key, 
                         allow_agent=False, look_for_keys=False)
          break
      except socket.error as e:
          print '.'
          time.sleep(5)
      except paramiko.AuthenticationException as e:
          print e
          time.sleep(5)
  channel = client.get_transport().open_session()
  channel.exec_command(cmd)
  exit_code = channel.recv_exit_status()
  output = channel.recv(1000000)
  client.close()
  return exit_code, output
870434
def map_return(self):
        """"""
        with Pool(self.cpu_count) as pool:
            vals = pool.map(self._func, self._iterable)
            pool.close()
            return vals
669741
def Main():
  """"""
  multiprocessing.freeze_support()

  tool = log2timeline_tool.Log2TimelineTool()

  if not tool.ParseArguments():
    return False

  if tool.show_info:
    tool.ShowInfo()
    return True

  if tool.show_troubleshooting:
    print('Using Python version {0!s}'.format(sys.version))
    print()
    print('Path: {0:s}'.format(os.path.abspath(__file__)))
    print()
    print(tool.GetVersionInformation())
    print()
    dependencies.CheckDependencies(verbose_output=True)

    print('Also see: https://plaso.readthedocs.io/en/latest/sources/user/'
          'Troubleshooting.html')
    return True

  have_list_option = False
  if tool.list_hashers:
    tool.ListHashers()
    have_list_option = True

  if tool.list_parsers_and_plugins:
    tool.ListParsersAndPlugins()
    have_list_option = True

  if tool.list_profilers:
    tool.ListProfilers()
    have_list_option = True

  if tool.list_timezones:
    tool.ListTimeZones()
    have_list_option = True

  if have_list_option:
    return True

  if tool.dependencies_check and not dependencies.CheckDependencies(
      verbose_output=False):
    return False

  try:
    tool.ExtractEventsFromSources()

  except (KeyboardInterrupt, errors.UserAbort):
    logging.warning('Aborted by user.')
    return False

  except (errors.BadConfigOption, errors.SourceScannerError) as exception:
    logging.warning(exception)
    return False

  return True
791258
def runInParallel(*fns):
        
        proc = []
        for fn in fns:
            p = Process(target=fn)
            p.start()
            proc.append(p)
        for p in proc:
            p.join()
395289
def process_shells_parallel(self, shells):
        """"""
        output = []
        success = True
        with closing(multiprocessing.Pool(multiprocessing.cpu_count())) as pool:
            for result in [Adapter(entry) for entry in pool.map(worker, [shell for shell in shells])]:
                output += result.output
                the_shell = [shell for shell in shells if shell['id'] == result.id][0]
                self.__handle_variable(the_shell['entry'], result.output)
                if not result.success:
                    success = False
        if success:
            self.logger.info("Parallel Processing Bash code: finished")
            return {'success': True, 'output': output}

        for line in self.run_cleanup(shells[0]['env'], 99):
            output.append(line)
        self.logger.error("Pipeline has failed: immediately leaving!")
        self.event.failed()
        return {'success': False, 'output': output}
15935
def delegate(self, fn, *args, **kwargs):
        """"""
        callback = functools.partial(fn, *args, **kwargs)
        coro = self.loop.run_in_executor(self.subexecutor, callback)
        return asyncio.ensure_future(coro)
828814
def processes(self):
        """"""
        if self._processes is None:
            self._processes = []
            for p in range(self.workers):
                t = Task(self._target, self._args, self._kwargs)
                t.name = "%s-%d" % (self.target_name, p)
                self._processes.append(t)

        return self._processes
292947
def _map_parallel(function, args, n_jobs):
    """"""
    # Following the error checking found in joblib
    multiprocessing = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None
    if multiprocessing:
        try:
            import multiprocessing
            import multiprocessing.pool
        except ImportError:
            multiprocessing = None
        if sys.platform.startswith("win") and PY2:
            msg = "Multiprocessing is not supported on Windows with Python 2.X. Setting n_jobs=1"
            logger.warning(msg)
            n_jobs = 1
    # 2nd stage: validate that locking is available on the system and
    #            issue a warning if not
    if multiprocessing:
        try:
            _sem = multiprocessing.Semaphore()
            del _sem  # cleanup
        except (ImportError, OSError) as e:
            multiprocessing = None
            logger.warning('{}. _map_parallel will operate in serial mode'.format(e))
    if multiprocessing and int(n_jobs) not in (0, 1):
        if n_jobs == -1:
            n_jobs = None
        try:
            pool = multiprocessing.Pool(processes=n_jobs)
            map_result = pool.map(function, args)
        finally:
            pool.close()
            pool.join()
    else:
        map_result = list(map(function, args))
    return map_result
28168
def run_in_thread_pool(self, pool_size=None, func=None, *args):
        """"""
        executor = Pool(pool_size)
        return self.loop.run_in_executor(executor, func, *args)
1367355
def benchmark_multi(optimizer):
    """"""
    # Get our benchmark stats
    all_stats = benchmark.compare(optimizer, PROBLEMS, runs=100)
    return benchmark.aggregate(all_stats)
95264
def run_in_process_pool(self, pool_size=None, func=None, *args):
        """"""
        executor = ProcessPool(pool_size)
        return self.loop.run_in_executor(executor, func, *args)
539600
def process_all(self, texts:Collection[str]) -> List[List[str]]:
        ""
        if self.n_cpus <= 1: return self._process_all_1(texts)
        with ProcessPoolExecutor(self.n_cpus) as e:
            return sum(e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), [])
1173117
def pool(n=None, dummy=False):
    

    if dummy:
        from multiprocessing.dummy import Pool
    else:
        from multiprocessing import Pool
    if n is None:
        import multiprocessing
        n = multiprocessing.cpu_count() - 1

    return Pool(n)
490586
def run_matrix_in_parallel(self, process_data):
        """"""
        worker_data = [{'matrix': entry, 'pipeline': process_data.pipeline,
                        'model': process_data.model, 'options': process_data.options,
                        'hooks': process_data.hooks} for entry in self.matrix
                       if Matrix.can_process_matrix(entry, process_data.options.matrix_tags)]
        output = []
        success = True
        with closing(multiprocessing.Pool(multiprocessing.cpu_count())) as pool:
            for result in pool.map(matrix_worker, worker_data):
                output += result['output']
                if not result['success']:
                    success = False
        return {'success': success, 'output': output}
388556
def _run_pipeline(self, multiprocessing):
        """"""
        if not self._flag_m:
            raise UnsupportedCall(f"Extractor(engine={self._exeng})' has no attribute '_run_pipline'")

        if not self._flag_q:
            self._flag_q = True
            warnings.warn("'Extractor(engine=pipeline)' does not support output; "
                          f"'fout={self._ofnm}' ignored", AttributeWarning, stacklevel=stacklevel())

        self._frnum = 1                                                 # frame number (revised)
        self._expkg = multiprocessing                                   # multiprocessing module
        self._mpprc = list()                                            # multiprocessing process list
        self._mpfdp = collections.defaultdict(multiprocessing.Queue)    # multiprocessing file pointer

        self._mpmng = multiprocessing.Manager()                         # multiprocessing manager
        self._mpkit = self._mpmng.Namespace()                           # multiprocessing work kit

        self._mpkit.counter = 0                                         # work count (on duty)
        self._mpkit.pool = 1                                            # work pool (ready)
        self._mpkit.current = 1                                         # current frame number
        self._mpkit.eof = False                                         # EOF flag
        self._mpkit.frames = dict()                                     # frame storage
        self._mpkit.trace = self._trace                                 # flow tracer
        self._mpkit.reassembly = copy.deepcopy(self._reasm)             # reassembly buffers

        # preparation
        self.record_header()
        self._mpfdp[0].put(self._gbhdr.length)

        # extraction
        while True:
            # check EOF
            if self._mpkit.eof:
                self._update_eof()
                break

            # check counter
            if self._mpkit.pool and self._mpkit.counter < CPU_CNT:
                # update file offset
                self._ifile.seek(self._mpfdp.pop(self._frnum-1).get(), os.SEEK_SET)

                # create worker
                # print(self._frnum, 'start')
                proc = multiprocessing.Process(
                    target=self._pipeline_read_frame,
                    kwargs={'mpkit': self._mpkit, 'mpfdp': self._mpfdp[self._frnum]}
                )

                # update status
                self._mpkit.pool -= 1
                self._mpkit.counter += 1

                # start and record
                proc.start()
                self._frnum += 1
                self._mpprc.append(proc)

            # check buffer
            if len(self._mpprc) >= CPU_CNT:
                [proc.join() for proc in self._mpprc[:-4]]
                del self._mpprc[:-4]
854933
def sort_by_mem_per_proc(self, reverse=False):
        """"""
        # Avoid sorting if mem_per_cpu is not available.
        if any(c.mem_per_proc > 0.0 for c in self):
            self._confs.sort(key=lambda c: c.mem_per_proc, reverse=reverse)
        return self
540283
def cpu_count() -> int:
    """"""
    if multiprocessing is None:
        return 1
    try:
        return multiprocessing.cpu_count()
    except NotImplementedError:
        pass
    try:
        return os.sysconf("SC_NPROCESSORS_CONF")
    except (AttributeError, ValueError):
        pass
    gen_log.error("Could not detect number of processors; assuming 1")
    return 1