Return all of the data in the minion datastore

    CLI Example:

    .. code-block:: bash

        salt '*' data.load
344591
def load():
    '''
    
    '''
    serial = salt.payload.Serial(__opts__)

    try:
        datastore_path = os.path.join(__opts__['cachedir'], 'datastore')
        with salt.utils.files.fopen(datastore_path, 'rb') as rfh:
            return serial.loads(rfh.read())
    except (IOError, OSError, NameError):
        return {}
124941
def load_data(self, key=None, **kwargs):
        
        self.store.open()
        if key:
            return pd.read_hdf(self.PATH_DATABASE, key, **kwargs)
        else:
            data_load = dict()
            for k in self.store.keys():
                k = k.replace('/', '')
                # **kwargs ej:= where=['index > 2009','index < 2010'],columns=['ordinal']
                data_load[k] = pd.read_hdf(self.PATH_DATABASE, k, **kwargs)
            self.data = data_load
        self.store.close()
        self.integridad_data()
19294
def ext_pillar(minion_id,  # pylint: disable=W0613
               pillar,  # pylint: disable=W0613
               command):
    '''
    
    '''
    try:
        command = command.replace('%s', minion_id)
        return deserialize(__salt__['cmd.run']('{0}'.format(command)))
    except Exception:
        log.critical('YAML data from %s failed to parse', command)
        return {}
425657
def dump(new_data):
    '''
    
    '''
    if not isinstance(new_data, dict):
        if isinstance(ast.literal_eval(new_data), dict):
            new_data = ast.literal_eval(new_data)
        else:
            return False

    try:
        datastore_path = os.path.join(__opts__['cachedir'], 'datastore')
        with salt.utils.files.fopen(datastore_path, 'w+b') as fn_:
            serial = salt.payload.Serial(__opts__)
            serial.dump(new_data, fn_)

        return True

    except (IOError, OSError, NameError):
        return False
655616
def get_load(jid):
    '''
    
    '''
    jid = _escape_jid(jid)
    conn = _get_conn()
    if conn is None:
        return None
    cur = conn.cursor()
    sql = '''SELECT jid, tgt_type, cmd, tgt, kwargs, ret, username, arg,''' \
          ''' fun FROM jids WHERE jid = %s'''
    cur.execute(sql, (jid,))
    data = cur.fetchone()
    if data:
        return _build_dict(data)
    _close_conn(conn)
    return {}
1281490
def get_load(jid):
    '''
    
    '''
    log.debug('sdstack_etcd returner <get_load> called jid: %s', jid)
    read_profile = __opts__.get('etcd.returner_read_profile')
    client, path = _get_conn(__opts__, read_profile)
    return salt.utils.json.loads(client.get('/'.join((path, 'jobs', jid, '.load.p'))).value)
1268658
def compile_pillar(self):
        '''
        
        '''
        load = {'id': self.minion_id,
                'grains': self.grains,
                'saltenv': self.opts['saltenv'],
                'pillarenv': self.opts['pillarenv'],
                'pillar_override': self.pillar_override,
                'extra_minion_data': self.extra_minion_data,
                'ver': '2',
                'cmd': '_pillar'}
        if self.ext:
            load['ext'] = self.ext
        try:
            ret_pillar = yield self.channel.crypted_transfer_decode_dictentry(
                load,
                dictkey='pillar',
            )
        except Exception:
            log.exception('Exception getting pillar:')
            raise SaltClientError('Exception getting pillar.')

        if not isinstance(ret_pillar, dict):
            msg = ('Got a bad pillar from master, type {0}, expecting dict: '
                   '{1}').format(type(ret_pillar).__name__, ret_pillar)
            log.error(msg)
            # raise an exception! Pillar isn't empty, we can't sync it!
            raise SaltClientError(msg)
        raise tornado.gen.Return(ret_pillar)
773183
def load(self, callback=None, errback=None, reload=False):
        
        if not reload and self.data:
            raise MonitorException('monitor already loaded')

        def success(result, *args):
            self.data = result
            if callback:
                return callback(self)
            else:
                return self

        return self._rest.retrieve(self.data['id'], callback=success,
                                   errback=errback)
768107
def _thread_return(cls, minion_instance, opts, data):
        '''
        
        '''
        fn_ = os.path.join(minion_instance.proc_dir, data['jid'])

        if opts['multiprocessing'] and not salt.utils.platform.is_windows():
            # Shutdown the multiprocessing before daemonizing
            salt.log.setup.shutdown_multiprocessing_logging()

            salt.utils.process.daemonize_if(opts)

            # Reconfigure multiprocessing logging after daemonizing
            salt.log.setup.setup_multiprocessing_logging()

        salt.utils.process.appendproctitle('{0}._thread_return {1}'.format(cls.__name__, data['jid']))

        sdata = {'pid': os.getpid()}
        sdata.update(data)
        log.info('Starting a new job %s with PID %s', data['jid'], sdata['pid'])
        with salt.utils.files.fopen(fn_, 'w+b') as fp_:
            fp_.write(minion_instance.serial.dumps(sdata))
        ret = {'success': False}
        function_name = data['fun']
        executors = data.get('module_executors') or \
                    getattr(minion_instance, 'module_executors', []) or \
                    opts.get('module_executors', ['direct_call'])
        allow_missing_funcs = any([
            minion_instance.executors['{0}.allow_missing_func'.format(executor)](function_name)
            for executor in executors
            if '{0}.allow_missing_func' in minion_instance.executors
        ])
        if function_name in minion_instance.functions or allow_missing_funcs is True:
            try:
                minion_blackout_violation = False
                if minion_instance.connected and minion_instance.opts['pillar'].get('minion_blackout', False):
                    whitelist = minion_instance.opts['pillar'].get('minion_blackout_whitelist', [])
                    # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist
                    if function_name != 'saltutil.refresh_pillar' and function_name not in whitelist:
                        minion_blackout_violation = True
                # use minion_blackout_whitelist from grains if it exists
                if minion_instance.opts['grains'].get('minion_blackout', False):
                    whitelist = minion_instance.opts['grains'].get('minion_blackout_whitelist', [])
                    if function_name != 'saltutil.refresh_pillar' and function_name not in whitelist:
                        minion_blackout_violation = True
                if minion_blackout_violation:
                    raise SaltInvocationError('Minion in blackout mode. Set \'minion_blackout\' '
                                             'to False in pillar or grains to resume operations. Only '
                                             'saltutil.refresh_pillar allowed in blackout mode.')

                if function_name in minion_instance.functions:
                    func = minion_instance.functions[function_name]
                    args, kwargs = load_args_and_kwargs(
                        func,
                        data['arg'],
                        data)
                else:
                    # only run if function_name is not in minion_instance.functions and allow_missing_funcs is True
                    func = function_name
                    args, kwargs = data['arg'], data
                minion_instance.functions.pack['__context__']['retcode'] = 0
                if isinstance(executors, six.string_types):
                    executors = [executors]
                elif not isinstance(executors, list) or not executors:
                    raise SaltInvocationError("Wrong executors specification: {0}. String or non-empty list expected".
                        format(executors))
                if opts.get('sudo_user', '') and executors[-1] != 'sudo':
                    executors[-1] = 'sudo'  # replace the last one with sudo
                log.trace('Executors list %s', executors)  # pylint: disable=no-member

                for name in executors:
                    fname = '{0}.execute'.format(name)
                    if fname not in minion_instance.executors:
                        raise SaltInvocationError("Executor '{0}' is not available".format(name))
                    return_data = minion_instance.executors[fname](opts, data, func, args, kwargs)
                    if return_data is not None:
                        break

                if isinstance(return_data, types.GeneratorType):
                    ind = 0
                    iret = {}
                    for single in return_data:
                        if isinstance(single, dict) and isinstance(iret, dict):
                            iret.update(single)
                        else:
                            if not iret:
                                iret = []
                            iret.append(single)
                        tag = tagify([data['jid'], 'prog', opts['id'], six.text_type(ind)], 'job')
                        event_data = {'return': single}
                        minion_instance._fire_master(event_data, tag)
                        ind += 1
                    ret['return'] = iret
                else:
                    ret['return'] = return_data

                retcode = minion_instance.functions.pack['__context__'].get(
                    'retcode',
                    salt.defaults.exitcodes.EX_OK
                )
                if retcode == salt.defaults.exitcodes.EX_OK:
                    # No nonzero retcode in __context__ dunder. Check if return
                    # is a dictionary with a "result" or "success" key.
                    try:
                        func_result = all(return_data.get(x, True)
                                          for x in ('result', 'success'))
                    except Exception:
                        # return data is not a dict
                        func_result = True
                    if not func_result:
                        retcode = salt.defaults.exitcodes.EX_GENERIC

                ret['retcode'] = retcode
                ret['success'] = retcode == salt.defaults.exitcodes.EX_OK
            except CommandNotFoundError as exc:
                msg = 'Command required for \'{0}\' not found'.format(
                    function_name
                )
                log.debug(msg, exc_info=True)
                ret['return'] = '{0}: {1}'.format(msg, exc)
                ret['out'] = 'nested'
                ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC
            except CommandExecutionError as exc:
                log.error(
                    'A command in \'%s\' had a problem: %s',
                    function_name, exc,
                    exc_info_on_loglevel=logging.DEBUG
                )
                ret['return'] = 'ERROR: {0}'.format(exc)
                ret['out'] = 'nested'
                ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC
            except SaltInvocationError as exc:
                log.error(
                    'Problem executing \'%s\': %s',
                    function_name, exc,
                    exc_info_on_loglevel=logging.DEBUG
                )
                ret['return'] = 'ERROR executing \'{0}\': {1}'.format(
                    function_name, exc
                )
                ret['out'] = 'nested'
                ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC
            except TypeError as exc:
                msg = 'Passed invalid arguments to {0}: {1}\n{2}'.format(
                    function_name, exc, func.__doc__ or ''
                )
                log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
                ret['return'] = msg
                ret['out'] = 'nested'
                ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC
            except Exception:
                msg = 'The minion function caused an exception'
                log.warning(msg, exc_info_on_loglevel=True)
                salt.utils.error.fire_exception(salt.exceptions.MinionError(msg), opts, job=data)
                ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc())
                ret['out'] = 'nested'
                ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC
        else:
            docs = minion_instance.functions['sys.doc']('{0}*'.format(function_name))
            if docs:
                docs[function_name] = minion_instance.functions.missing_fun_string(function_name)
                ret['return'] = docs
            else:
                ret['return'] = minion_instance.functions.missing_fun_string(function_name)
                mod_name = function_name.split('.')[0]
                if mod_name in minion_instance.function_errors:
                    ret['return'] += ' Possible reasons: \'{0}\''.format(
                        minion_instance.function_errors[mod_name]
                    )
            ret['success'] = False
            ret['retcode'] = salt.defaults.exitcodes.EX_GENERIC
            ret['out'] = 'nested'

        ret['jid'] = data['jid']
        ret['fun'] = data['fun']
        ret['fun_args'] = data['arg']
        if 'master_id' in data:
            ret['master_id'] = data['master_id']
        if 'metadata' in data:
            if isinstance(data['metadata'], dict):
                ret['metadata'] = data['metadata']
            else:
                log.warning('The metadata parameter must be a dictionary. Ignoring.')
        if minion_instance.connected:
            minion_instance._return_pub(
                ret,
                timeout=minion_instance._return_retry_timer()
            )

        # Add default returners from minion config
        # Should have been coverted to comma-delimited string already
        if isinstance(opts.get('return'), six.string_types):
            if data['ret']:
                data['ret'] = ','.join((data['ret'], opts['return']))
            else:
                data['ret'] = opts['return']

        log.debug('minion return: %s', ret)
        # TODO: make a list? Seems odd to split it this late :/
        if data['ret'] and isinstance(data['ret'], six.string_types):
            if 'ret_config' in data:
                ret['ret_config'] = data['ret_config']
            if 'ret_kwargs' in data:
                ret['ret_kwargs'] = data['ret_kwargs']
            ret['id'] = opts['id']
            for returner in set(data['ret'].split(',')):
                try:
                    returner_str = '{0}.returner'.format(returner)
                    if returner_str in minion_instance.returners:
                        minion_instance.returners[returner_str](ret)
                    else:
                        returner_err = minion_instance.returners.missing_fun_string(returner_str)
                        log.error(
                            'Returner %s could not be loaded: %s',
                            returner_str, returner_err
                        )
                except Exception as exc:
                    log.exception(
                        'The return failed for job %s: %s', data['jid'], exc
                    )
325841
def get_load(jid):
    '''
    
    '''
    jid_dir = salt.utils.jid.jid_dir(jid, _job_dir(), __opts__['hash_type'])
    load_fn = os.path.join(jid_dir, LOAD_P)
    if not os.path.exists(jid_dir) or not os.path.exists(load_fn):
        return {}
    serial = salt.payload.Serial(__opts__)
    ret = {}
    load_p = os.path.join(jid_dir, LOAD_P)
    num_tries = 5
    for index in range(1, num_tries + 1):
        with salt.utils.files.fopen(load_p, 'rb') as rfh:
            try:
                ret = serial.load(rfh)
                break
            except Exception as exc:
                if index == num_tries:
                    time.sleep(0.25)
    else:
        log.critical('Failed to unpack %s', load_p)
        raise exc
    if ret is None:
        ret = {}
    minions_cache = [os.path.join(jid_dir, MINIONS_P)]
    minions_cache.extend(
        glob.glob(os.path.join(jid_dir, SYNDIC_MINIONS_P.format('*')))
    )
    all_minions = set()
    for minions_path in minions_cache:
        log.debug('Reading minion list from %s', minions_path)
        try:
            with salt.utils.files.fopen(minions_path, 'rb') as rfh:
                all_minions.update(serial.load(rfh))
        except IOError as exc:
            salt.utils.files.process_read_exception(exc, minions_path)

    if all_minions:
        ret['Minions'] = sorted(all_minions)

    return ret
1015131
def _load(formula):
    '''
    
    '''

    # Compute possibilities
    _mk_client()
    paths = []
    for ext in ('yaml', 'json'):
        source_url = salt.utils.url.create(formula + '/defaults.' + ext)
        paths.append(source_url)
    # Fetch files from master
    defaults_files = __context__['cp.fileclient'].cache_files(paths)

    for file_ in defaults_files:
        if not file_:
            # Skip empty string returned by cp.fileclient.cache_files.
            continue

        suffix = file_.rsplit('.', 1)[-1]
        if suffix == 'yaml':
            loader = salt.utils.yaml.safe_load
        elif suffix == 'json':
            loader = salt.utils.json.load
        else:
            log.debug("Failed to determine loader for %r", file_)
            continue

        if os.path.exists(file_):
            log.debug("Reading defaults from %r", file_)
            with salt.utils.files.fopen(file_) as fhr:
                defaults = loader(fhr)
                log.debug("Read defaults %r", defaults)

            return defaults or {}
243943
def get_fun(fun):
    '''
    
    '''
    conn = _get_conn(ret=None)
    cur = conn.cursor()
    sql = '''SELECT s.id,s.jid, s.full_ret
            FROM salt_returns s
            JOIN ( SELECT MAX(jid) AS jid FROM salt_returns GROUP BY fun, id) max
            ON s.jid = max.jid
            WHERE s.fun = ?
            '''

    cur.execute(sql, (fun,))
    data = cur.fetchall()

    ret = {}
    if data:
        for minion, _, retval in data:
            ret[minion] = salt.utils.json.loads(retval)
    _close_conn(conn)
    return ret
274238
def ext_pillar(minion_id, pillar, *args, **kwargs):
    '''
    
    '''
    # Node definitions path will be retrieved from args (or set to default),
    # then added to 'salt_data' dict that is passed to the 'get_pillars'
    # function. The dictionary contains:
    #     - __opts__
    #     - __salt__
    #     - __grains__
    #     - __pillar__
    #     - minion_id
    #     - path
    #
    # If successful, the function will return a pillar dict for minion_id.

    # If path has not been set, make a default
    for i in args:
        if 'path' not in i:
            path = '/srv/saltclass'
            args[i]['path'] = path
            log.warning('path variable unset, using default: %s', path)
        else:
            path = i['path']

    # Create a dict that will contain our salt dicts to pass it to reclass
    salt_data = {
        '__opts__': __opts__,
        '__salt__': __salt__,
        '__grains__': __grains__,
        '__pillar__': pillar,
        'minion_id': minion_id,
        'path': path
    }

    return sc.get_pillars(minion_id, salt_data)
627215
def load_modules(self, data=None, proxy=None):
        '''
        
        '''
        log.info('Loading fresh modules for state activity')
        # Load a modified client interface that looks like the interface used
        # from the minion, but uses remote execution
        #
        self.functions = salt.client.FunctionWrapper(
                self.opts,
                self.opts['id']
                )
        # Load the states, but they should not be used in this class apart
        # from inspection
        self.utils = salt.loader.utils(self.opts)
        self.serializers = salt.loader.serializers(self.opts)
        self.states = salt.loader.states(self.opts, self.functions, self.utils, self.serializers)
        self.rend = salt.loader.render(self.opts, self.functions, states=self.states, context=self.state_con)
457996
def load_modules(self, data=None, proxy=None):
        '''
        
        '''
        log.info('Loading fresh modules for state activity')
        self.utils = salt.loader.utils(self.opts)
        self.functions = salt.loader.minion_mods(self.opts, self.state_con,
                                                 utils=self.utils,
                                                 proxy=self.proxy)
        if isinstance(data, dict):
            if data.get('provider', False):
                if isinstance(data['provider'], six.string_types):
                    providers = [{data['state']: data['provider']}]
                elif isinstance(data['provider'], list):
                    providers = data['provider']
                else:
                    providers = {}
                for provider in providers:
                    for mod in provider:
                        funcs = salt.loader.raw_mod(self.opts,
                                provider[mod],
                                self.functions)
                        if funcs:
                            for func in funcs:
                                f_key = '{0}{1}'.format(
                                        mod,
                                        func[func.rindex('.'):]
                                        )
                                self.functions[f_key] = funcs[func]
        self.serializers = salt.loader.serializers(self.opts)
        self._load_states()
        self.rend = salt.loader.render(self.opts, self.functions,
                                       states=self.states, proxy=self.proxy, context=self.state_con)
1089004
def fetch(self,
              minion_id,
              pillar,  # pylint: disable=W0613
              *args,
              **kwargs):
        '''
        
        '''
        db_name = self._db_name()
        log.info('Querying %s for information for %s', db_name, minion_id)
        #
        #    log.debug('ext_pillar %s args: %s', db_name, args)
        #    log.debug('ext_pillar %s kwargs: %s', db_name, kwargs)
        #
        # Most of the heavy lifting is in this class for ease of testing.
        qbuffer = self.extract_queries(args, kwargs)
        with self._get_cursor() as cursor:
            for root, details in qbuffer:
                # Run the query
                cursor.execute(details['query'], (minion_id,))

                # Extract the field names the db has returned and process them
                self.process_fields([row[0] for row in cursor.description], details['depth'])
                self.enter_root(root)
                self.as_list = details['as_list']
                if details['with_lists']:
                    self.with_lists = details['with_lists']
                else:
                    self.with_lists = []
                self.ignore_null = details['ignore_null']
                self.process_results(cursor.fetchall())

                log.debug('ext_pillar %s: Return data: %s', db_name, self)
        return self.result
1369658
def get_load(jid):
    '''
    
    '''
    ret = {}
    for returner_ in __opts__[CONFIG_KEY]:
        ret.update(_mminion().returners['{0}.get_load'.format(returner_)](jid))

    return ret