Add multiprocessing guards.

        Forces a connection to be reconnected if it is detected
        as having been shared to a sub-process.
503891
def _add_process_guards(self, engine):
        

        @sqlalchemy.event.listens_for(engine, "connect")
        def connect(dbapi_connection, connection_record):
            connection_record.info['pid'] = os.getpid()

        @sqlalchemy.event.listens_for(engine, "checkout")
        def checkout(dbapi_connection, connection_record, connection_proxy):
            pid = os.getpid()
            if connection_record.info['pid'] != pid:
                self.logger.debug(
                    "Parent process %(orig)s forked (%(newproc)s) with an open database connection, "
                    "which is being discarded and recreated." % {"newproc": pid, "orig": connection_record.info['pid']})
                connection_record.connection = connection_proxy.connection = None
                raise exc.DisconnectionError(
                    "Connection record belongs to pid %s, attempting to check out in pid %s" % (
                        connection_record.info['pid'], pid)
                )
956104
def init(opts):
    '''
    
    '''
    proxy_dict = opts.get('proxy', {})
    conn_args = proxy_dict.copy()
    conn_args.pop('proxytype', None)
    opts['multiprocessing'] = conn_args.get('multiprocessing', True)
    # This is not a SSH-based proxy, so it should be safe to enable
    # multiprocessing.
    try:
        conn = pyeapi.client.connect(**conn_args)
        node = pyeapi.client.Node(conn, enablepwd=conn_args.get('enablepwd'))
        pyeapi_device['connection'] = node
        pyeapi_device['initialized'] = True
        pyeapi_device['up'] = True
    except pyeapi.eapilib.ConnectionError as cerr:
        log.error('Unable to connect to %s', conn_args['host'], exc_info=True)
        return False
    return True
891217
def mp_check_impl(self, process_count):
        

        from multiprocessing import Process, Queue

        options = self.reporter.options

        # this is the function that will be run in a separate process,
        # which will handle the tasks queue and feed into the results
        # queue
        func = _mp_run_check

        # normally this would happen lazily, but since we'll have
        # multiple processes all running reports at the same time, we
        # need to make sure the setup is done before-hand. This is
        # hackish, but in particular this keeps the HTML reports from
        # trying to perform the default data copy over and over.
        self.reporter.setup()

        # enqueue the sub-reports for multi-processing. Other types of
        # changes can happen sync.
        changes = list(self.collect_impl())

        task_count = 0
        tasks = Queue()
        results = Queue()

        try:
            # as soon as we start using the tasks queue, we need to be
            # catching the KeyboardInterrupt event so that we can
            # drain the queue and lets its underlying thread terminate
            # happily.

            # TODO: is there a better way to handle this shutdown
            # gracefully?

            # feed any sub-reports to the tasks queue
            for index in range(0, len(changes)):
                change = changes[index]
                if isinstance(change, (DistJarReport, DistClassReport)):
                    changes[index] = None
                    tasks.put((index, change))
                    task_count += 1

            # infrequent edge case, but don't bother starting more
            # helpers than we'll ever use
            process_count = min(process_count, task_count)

            # start the number of helper processes, and make sure
            # there are that many stop sentinels at the end of the
            # tasks queue
            for _i in range(0, process_count):
                tasks.put(None)
                process = Process(target=func, args=(tasks, results, options))
                process.daemon = False
                process.start()

            # while the helpers are running, perform our checks
            for change in changes:
                if change:
                    change.check()

            # get all of the results and feed them back into our change
            for _i in range(0, task_count):
                index, change = results.get()
                changes[index] = change

        except KeyboardInterrupt:
            # drain the tasks queue so it will exit gracefully
            for _change in iter(tasks.get, None):
                pass
            raise

        # complete the check by setting our internal collection of
        # child changes and returning our overall status
        c = False
        for change in changes:
            c = c or change.is_change()
        self.changes = changes
        return c, None
629289
def reconnect(self):
		""""""
		self.lock.acquire()
		if self.use_ssl:
			self.client = http.client.HTTPSConnection(self.host, self.port, context=self.ssl_context)
		else:
			self.client = http.client.HTTPConnection(self.host, self.port)
		self.lock.release()
329878
def proc_check_guard(self, instance, sql):
        
        self.open_db_connections(instance, self.PROC_GUARD_DB_KEY)
        cursor = self.get_cursor(instance, self.PROC_GUARD_DB_KEY)

        should_run = False
        try:
            cursor.execute(sql, ())
            result = cursor.fetchone()
            should_run = result[0] == 1
        except Exception as e:
            self.log.error("Failed to run proc_only_if sql {} : {}".format(sql, e))

        self.close_cursor(cursor)
        self.close_db_connections(instance, self.PROC_GUARD_DB_KEY)
        return should_run
1336226
def _connectRelay(self, process, protocolFactory):
        
        try:
            wf = _WrappingFactory(protocolFactory)
            connector = RelayConnector(process, wf, self.timeout,
                                       self.inductor.reactor)
            connector.connect()
        except:
            return defer.fail()
        # Return a deferred that is called back when the protocol is connected.
        return wf._onConnection
252393
def addSubprocess(self, fds, name, factory):
        
        self._lock.run(self._addSubprocess, self, fds, name, factory)
1373199
def install_mp_handler(logger=None):
    
    if logger is None:
        logger = logging.getLogger()

    for i, orig_handler in enumerate(list(logger.handlers)):
        handler = MultiProcessingHandler(
            'mp-handler-{0}'.format(i), sub_handler=orig_handler)

        logger.removeHandler(orig_handler)
        logger.addHandler(handler)
80991
def pre_fork(self, process_manager):
        '''
        
        '''
        salt.transport.mixins.auth.AESReqServerMixin.pre_fork(self, process_manager)
        if USE_LOAD_BALANCER:
            self.socket_queue = multiprocessing.Queue()
            process_manager.add_process(
                LoadBalancerServer, args=(self.opts, self.socket_queue)
            )
        elif not salt.utils.platform.is_windows():
            self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            _set_tcp_keepalive(self._socket, self.opts)
            self._socket.setblocking(0)
            self._socket.bind((self.opts['interface'], int(self.opts['ret_port'])))
281537
def start_proc_mask_signal(proc):
    
    if not isinstance(proc, list):
        proc = [proc]

    with mask_sigint():
        for p in proc:
            if isinstance(p, mp.Process):
                if sys.version_info < (3, 4) or mp.get_start_method() == 'fork':
                    log_once(
"Starting a process with 'fork' method is not safe and may consume unnecessary extra memory."
" Use 'forkserver' method (available after Py3.4) instead if you run into any issues. "
"See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods",
'warn')  # noqa
            p.start()
829162
def setup_multiprocessing_logging(queue=None):
    '''
    
    '''
    from salt.utils.platform import is_windows

    global __MP_LOGGING_CONFIGURED
    global __MP_LOGGING_QUEUE_HANDLER

    if __MP_IN_MAINPROCESS is True and not is_windows():
        # We're in the MainProcess, return! No multiprocessing logging setup shall happen
        # Windows is the exception where we want to set up multiprocessing
        # logging in the MainProcess.
        return

    try:
        logging._acquireLock()  # pylint: disable=protected-access

        if __MP_LOGGING_CONFIGURED is True:
            return

        # Let's set it to true as fast as possible
        __MP_LOGGING_CONFIGURED = True

        if __MP_LOGGING_QUEUE_HANDLER is not None:
            return

        # The temp null and temp queue logging handlers will store messages.
        # Since noone will process them, memory usage will grow. If they
        # exist, remove them.
        __remove_null_logging_handler()
        __remove_queue_logging_handler()

        # Let's add a queue handler to the logging root handlers
        __MP_LOGGING_QUEUE_HANDLER = SaltLogQueueHandler(queue or get_multiprocessing_logging_queue())
        logging.root.addHandler(__MP_LOGGING_QUEUE_HANDLER)
        # Set the logging root level to the lowest needed level to get all
        # desired messages.
        log_level = get_multiprocessing_logging_level()
        logging.root.setLevel(log_level)
        logging.getLogger(__name__).debug(
            'Multiprocessing queue logging configured for the process running '
            'under PID: %s at log level %s', os.getpid(), log_level
        )
        # The above logging call will create, in some situations, a futex wait
        # lock condition, probably due to the multiprocessing Queue's internal
        # lock and semaphore mechanisms.
        # A small sleep will allow us not to hit that futex wait lock condition.
        time.sleep(0.0001)
    finally:
        logging._releaseLock()
1215468
def add_process(self, tgt, args=None, kwargs=None, name=None):
        '''
        
        '''
        if args is None:
            args = []

        if kwargs is None:
            kwargs = {}

        if salt.utils.platform.is_windows():
            # Need to ensure that 'log_queue' and 'log_queue_level' is
            # correctly transferred to processes that inherit from
            # 'MultiprocessingProcess'.
            if type(MultiprocessingProcess) is type(tgt) and (
                    issubclass(tgt, MultiprocessingProcess)):
                need_log_queue = True
            else:
                need_log_queue = False

            if need_log_queue:
                if 'log_queue' not in kwargs:
                    if hasattr(self, 'log_queue'):
                        kwargs['log_queue'] = self.log_queue
                    else:
                        kwargs['log_queue'] = (
                            salt.log.setup.get_multiprocessing_logging_queue()
                        )
                if 'log_queue_level' not in kwargs:
                    if hasattr(self, 'log_queue_level'):
                        kwargs['log_queue_level'] = self.log_queue_level
                    else:
                        kwargs['log_queue_level'] = (
                            salt.log.setup.get_multiprocessing_logging_level()
                        )

        # create a nicer name for the debug log
        if name is None:
            if isinstance(tgt, types.FunctionType):
                name = '{0}.{1}'.format(
                    tgt.__module__,
                    tgt.__name__,
                )
            else:
                name = '{0}{1}.{2}'.format(
                    tgt.__module__,
                    '.{0}'.format(tgt.__class__) if six.text_type(tgt.__class__) != "<type 'type'>" else '',
                    tgt.__name__,
                )

        if type(multiprocessing.Process) is type(tgt) and issubclass(tgt, multiprocessing.Process):
            process = tgt(*args, **kwargs)
        else:
            process = multiprocessing.Process(target=tgt, args=args, kwargs=kwargs, name=name)

        if isinstance(process, SignalHandlingMultiprocessingProcess):
            with default_signals(signal.SIGINT, signal.SIGTERM):
                process.start()
        else:
            process.start()
        log.debug("Started '%s' with pid %s", name, process.pid)
        self._process_map[process.pid] = {'tgt': tgt,
                                          'args': args,
                                          'kwargs': kwargs,
                                          'Process': process}
        return process
487536
def add(self, connection):
        ''''''
        key = (connection.host, connection.port)
        with self._lock:
            if key not in self._connections:
                self._connections[key] = connection
                self.added(connection)
                return connection
            else:
                return None
741017
def add(self, nb = 1, name = None):
        
        for x in xrange(nb):
            self.count_lock.acquire()
            self.shared['workers'] += 1
            xid = self.shared['workers']
            self.kill_event.clear()
            self.count_lock.release()
            w   = WorkerProc(xid, self.queue, self.shared, self.tasks, self.life_time, self.count_lock, self.kill_event)
            w.name = self.get_name(xid, name)
            w.start()
323884
def connectExec(connection, protocol, commandLine):
    
    deferred = connectSession(connection, protocol)
    @deferred.addCallback
    def requestSubsystem(session):
        return session.requestExec(commandLine)
    return deferred
148145
def release(self, connection):
        """"""
        self._checkpid()
        if connection.pid == self.pid:
            idx = connection._pattern_idx
            self._in_use_connections[idx].remove(connection)
            self._available_connections[idx].append(connection)
561392
def __join_connections(self):
        

        interval_s = nsq.config.client.CONNECTION_CLOSE_AUDIT_WAIT_S
        graceful_wait_s = nsq.config.client.CONNECTION_QUIT_CLOSE_TIMEOUT_S
        graceful = False

        while graceful_wait_s > 0:
            if not self.__connections:
                break

            connected_list = [c.is_connected for (n, c, g) in self.__connections]
            if any(connected_list) is False:
                graceful = True
                break

            # We need to give the greenlets periodic control, in order to finish 
            # up.

            gevent.sleep(interval_s)
            graceful_wait_s -= interval_s

        if graceful is False:
            connected_list = [c for (n, c, g) in self.__connections if c.is_connected]
            _logger.error("We were told to terminate, but not all "
                          "connections were stopped: [%s]", connected_list)