Pull salt file references out of the states
351837
def salt_refs(data, ret=None):
    '''
    
    '''
    proto = 'salt://'
    if ret is None:
        ret = []
    if isinstance(data, six.string_types):
        if data.startswith(proto) and data not in ret:
            ret.append(data)
    if isinstance(data, list):
        for comp in data:
            salt_refs(comp, ret)
    if isinstance(data, dict):
        for comp in data:
            salt_refs(data[comp], ret)
    return ret
1025536
def get_chunks(self, exclude=None, whitelist=None):
        '''
        
        '''
        ret = {}
        err = []
        try:
            top = self.get_top()
        except SaltRenderError as err:
            return ret
        except Exception:
            trb = traceback.format_exc()
            err.append(trb)
            return err
        err += self.verify_tops(top)
        matches = self.top_matches(top)
        if not matches:
            msg = 'No Top file found!'
            raise SaltRenderError(msg)
        matches = self.matches_whitelist(matches, whitelist)
        high, errors = self.render_highstate(matches)
        if exclude:
            if isinstance(exclude, six.string_types):
                exclude = exclude.split(',')
            if '__exclude__' in high:
                high['__exclude__'].extend(exclude)
            else:
                high['__exclude__'] = exclude
            err += errors
        high, ext_errors = self.state.reconcile_extend(high)
        err += ext_errors
        err += self.state.verify_high(high)
        if err:
            raise SaltRenderError(err)
        return self.state.compile_high_data(high)
1195299
function HKDF_extract(salt, ikm) {
  keylog('salt', salt);
  keylog('ikm', ikm);
  return keylog('extract', HMAC_hash(salt, ikm));
}
1361284
def _get_deps(deps, tree_base, saltenv='base'):
    '''
    
    '''
    deps_list = ''
    if deps is None:
        return deps_list
    if not isinstance(deps, list):
        raise SaltInvocationError(
            '\'deps\' must be a Python list or comma-separated string'
        )
    for deprpm in deps:
        parsed = _urlparse(deprpm)
        depbase = os.path.basename(deprpm)
        dest = os.path.join(tree_base, depbase)
        if parsed.scheme:
            __salt__['cp.get_url'](deprpm, dest, saltenv=saltenv)
        else:
            shutil.copy(deprpm, dest)

        deps_list += ' {0}'.format(dest)

    return deps_list
444152
def _parse_state_file(state_file_path='terraform.tfstate'):
    '''
    
    '''
    ret = {}
    with salt.utils.files.fopen(state_file_path, 'r') as fh_:
        tfstate = salt.utils.json.load(fh_)

    modules = tfstate.get('modules')
    if not modules:
        log.error('Malformed tfstate file. No modules found')
        return ret

    for module in modules:
        resources = module.get('resources', [])
        for resource_name, resource in salt.ext.six.iteritems(resources):
            roster_entry = None
            if resource['type'] == 'salt_host':
                roster_entry = _handle_salt_host_resource(resource)

            if not roster_entry:
                continue

            minion_id = roster_entry.get(MINION_ID, resource.get('id'))
            if not minion_id:
                continue

            if MINION_ID in roster_entry:
                del roster_entry[MINION_ID]
            _add_ssh_key(roster_entry)
            ret[minion_id] = roster_entry
    return ret
819519
def _cl_gof3r(file_info, region):
        """"""
        command = ["gof3r", "get", "--no-md5",
                   "-k", file_info.key,
                   "-b", file_info.bucket]
        if region != "us-east-1":
            command += ["--endpoint=s3-%s.amazonaws.com" % region]
        return (command, "gof3r")
57767
def _normalize_roots(file_roots):
    '''
    
    '''
    for saltenv, dirs in six.iteritems(file_roots):
        normalized_saltenv = six.text_type(saltenv)
        if normalized_saltenv != saltenv:
            file_roots[normalized_saltenv] = file_roots.pop(saltenv)
        if not isinstance(dirs, (list, tuple)):
            file_roots[normalized_saltenv] = []
        file_roots[normalized_saltenv] = \
                _expand_glob_path(file_roots[normalized_saltenv])
    return file_roots
1217304
def rmR(kls, path):
		""""""
		if   os.path.islink(path) or os.path.isfile(path):
			os.unlink(path)
		elif os.path.isdir(path):
			walker = os.walk(path, topdown=False, followlinks=False)
			for dirpath, dirnames, filenames in walker:
				for f in filenames:
					os.unlink(os.path.join(dirpath, f))
				for d in dirnames:
					os.rmdir (os.path.join(dirpath, d))
			os.rmdir(path)
443860
def _file_lists(load, form):
    '''
    
    '''
    if 'env' in load:
        # "env" is not supported; Use "saltenv".
        load.pop('env')

    saltenv = load['saltenv']
    if saltenv not in __opts__['file_roots']:
        if '__env__' in __opts__['file_roots']:
            log.debug("salt environment '%s' maps to __env__ file_roots directory", saltenv)
            saltenv = '__env__'
        else:
            return []

    list_cachedir = os.path.join(__opts__['cachedir'], 'file_lists', 'roots')
    if not os.path.isdir(list_cachedir):
        try:
            os.makedirs(list_cachedir)
        except os.error:
            log.critical('Unable to make cachedir %s', list_cachedir)
            return []
    list_cache = os.path.join(list_cachedir, '{0}.p'.format(salt.utils.files.safe_filename_leaf(saltenv)))
    w_lock = os.path.join(list_cachedir, '.{0}.w'.format(salt.utils.files.safe_filename_leaf(saltenv)))
    cache_match, refresh_cache, save_cache = \
        salt.fileserver.check_file_list_cache(
            __opts__, form, list_cache, w_lock
        )
    if cache_match is not None:
        return cache_match
    if refresh_cache:
        ret = {
            'files': set(),
            'dirs': set(),
            'empty_dirs': set(),
            'links': {}
        }

        def _add_to(tgt, fs_root, parent_dir, items):
            '''
            Add the files to the target set
            '''
            def _translate_sep(path):
                '''
                Translate path separators for Windows masterless minions
                '''
                return path.replace('\\', '/') if os.path.sep == '\\' else path

            for item in items:
                abs_path = os.path.join(parent_dir, item)
                log.trace('roots: Processing %s', abs_path)
                is_link = salt.utils.path.islink(abs_path)
                log.trace(
                    'roots: %s is %sa link',
                    abs_path, 'not ' if not is_link else ''
                )
                if is_link and __opts__['fileserver_ignoresymlinks']:
                    continue
                rel_path = _translate_sep(os.path.relpath(abs_path, fs_root))
                log.trace('roots: %s relative path is %s', abs_path, rel_path)
                if salt.fileserver.is_file_ignored(__opts__, rel_path):
                    continue
                tgt.add(rel_path)
                try:
                    if not os.listdir(abs_path):
                        ret['empty_dirs'].add(rel_path)
                except Exception:
                    # Generic exception because running os.listdir() on a
                    # non-directory path raises an OSError on *NIX and a
                    # WindowsError on Windows.
                    pass
                if is_link:
                    link_dest = salt.utils.path.readlink(abs_path)
                    log.trace(
                        'roots: %s symlink destination is %s',
                        abs_path, link_dest
                    )
                    if salt.utils.platform.is_windows() \
                            and link_dest.startswith('\\\\'):
                        # Symlink points to a network path. Since you can't
                        # join UNC and non-UNC paths, just assume the original
                        # path.
                        log.trace(
                            'roots: %s is a UNC path, using %s instead',
                            link_dest, abs_path
                        )
                        link_dest = abs_path
                    if link_dest.startswith('..'):
                        joined = os.path.join(abs_path, link_dest)
                    else:
                        joined = os.path.join(
                            os.path.dirname(abs_path), link_dest
                        )
                    rel_dest = _translate_sep(
                        os.path.relpath(
                            os.path.realpath(os.path.normpath(joined)),
                            fs_root
                        )
                    )
                    log.trace(
                        'roots: %s relative path is %s',
                        abs_path, rel_dest
                    )
                    if not rel_dest.startswith('..'):
                        # Only count the link if it does not point
                        # outside of the root dir of the fileserver
                        # (i.e. the "path" variable)
                        ret['links'][rel_path] = link_dest

        for path in __opts__['file_roots'][saltenv]:
            for root, dirs, files in salt.utils.path.os_walk(
                    path,
                    followlinks=__opts__['fileserver_followsymlinks']):
                _add_to(ret['dirs'], path, root, dirs)
                _add_to(ret['files'], path, root, files)

        ret['files'] = sorted(ret['files'])
        ret['dirs'] = sorted(ret['dirs'])
        ret['empty_dirs'] = sorted(ret['empty_dirs'])

        if save_cache:
            try:
                salt.fileserver.write_file_list_cache(
                    __opts__, ret, list_cache, w_lock
                )
            except NameError:
                # Catch msgpack error in salt-ssh
                pass
        return ret.get(form, [])
    # Shouldn't get here, but if we do, this prevents a TypeError
    return []
950832
def get_dir(path, dest, saltenv='base'):
    '''
    
    '''
    src = __context__['fileclient'].cache_dir(
        path,
        saltenv,
        cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_']))
    src = ' '.join(src)
    single = salt.client.ssh.Single(
            __opts__,
            '',
            **__salt__.kwargs)
    ret = single.shell.send(src, dest)
    return not ret[2]
878331
def _get_local_ref(region, ref_file, out_vcf_base):
    
    out_file = "{0}.fa".format(out_vcf_base)
    if not file_exists(out_file):
        with pysam.Fastafile(ref_file) as in_pysam:
            contig, start, end = region
            seq = in_pysam.fetch(contig, int(start), int(end))
            with open(out_file, "w") as out_handle:
                out_handle.write(">{0}-{1}-{2}\n{3}".format(contig, start, end,
                                                              str(seq)))
    with open(out_file) as in_handle:
        in_handle.readline()
        size = len(in_handle.readline().strip())
    return out_file, size
190302
def _get_state(pkg):
    '''
    
    '''
    cmd = ['opkg', 'status']
    cmd.append(pkg)
    out = __salt__['cmd.run'](cmd, python_shell=False)
    state_flag = ''
    for line in salt.utils.itertools.split(out, '\n'):
        if line.startswith('Status'):
            _status, _state_want, state_flag, _state_status = line.split()

    return state_flag
34998
def sls(mods, saltenv='base', test=None, exclude=None, **kwargs):
    '''
    
    '''
    st_kwargs = __salt__.kwargs
    __opts__['grains'] = __grains__
    __pillar__.update(kwargs.get('pillar', {}))
    opts = salt.utils.state.get_sls_opts(__opts__, **kwargs)
    st_ = salt.client.ssh.state.SSHHighState(
            opts,
            __pillar__,
            __salt__,
            __context__['fileclient'])
    st_.push_active()
    mods = _parse_mods(mods)
    high_data, errors = st_.render_highstate({saltenv: mods})
    if exclude:
        if isinstance(exclude, six.string_types):
            exclude = exclude.split(',')
        if '__exclude__' in high_data:
            high_data['__exclude__'].extend(exclude)
        else:
            high_data['__exclude__'] = exclude
    high_data, ext_errors = st_.state.reconcile_extend(high_data)
    errors += ext_errors
    errors += st_.state.verify_high(high_data)
    if errors:
        return errors
    high_data, req_in_errors = st_.state.requisite_in(high_data)
    errors += req_in_errors
    high_data = st_.state.apply_exclude(high_data)
    # Verify that the high data is structurally sound
    if errors:
        return errors
    # Compile and verify the raw chunks
    chunks = st_.state.compile_high_data(high_data)
    file_refs = salt.client.ssh.state.lowstate_file_refs(
            chunks,
            _merge_extra_filerefs(
                kwargs.get('extra_filerefs', ''),
                opts.get('extra_filerefs', '')
                )
            )

    roster = salt.roster.Roster(opts, opts.get('roster', 'flat'))
    roster_grains = roster.opts['grains']

    # Create the tar containing the state pkg and relevant files.
    _cleanup_slsmod_low_data(chunks)
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __context__['fileclient'],
            chunks,
            file_refs,
            __pillar__,
            st_kwargs['id_'],
            roster_grains)
    trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, opts['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            opts['thin_dir'],
            test,
            trans_tar_sum,
            opts['hash_type'])
    single = salt.client.ssh.Single(
            opts,
            cmd,
            fsclient=__context__['fileclient'],
            minion_opts=__salt__.minion_opts,
            **st_kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(opts['thin_dir']))
    stdout, stderr, _ = single.cmd_block()

    # Clean up our tar
    try:
        os.remove(trans_tar)
    except (OSError, IOError):
        pass

    # Read in the JSON data and return the data structure
    try:
        return salt.utils.json.loads(stdout)
    except Exception as e:
        log.error("JSON Render failed for: %s\n%s", stdout, stderr)
        log.error(six.text_type(e))

    # If for some reason the json load fails, return the stdout
    return stdout
1300297
function() {

    return [
      {
        type: 'list',
        name: 'database',
        message: 'Pull database from which environment?',
        choices: function() {

          // Get approved choices
          var choices = getEnvPullChoices();

          // Add none choice
          choices.push({
            name: 'Do not pull a database',
            value: 'none'
          });

          // Return our choices
          return choices;

        },
        default: function() {
          return pantheonConf.env;
        }
      },
      {
        type: 'list',
        name: 'files',
        message: 'Pull files from which environment?',
        choices: function() {

          // Get approved choices
          var choices = getEnvPullChoices();

          // Add none choice
          choices.push({
            name: 'Do not pull files',
            value: 'none'
          });

          // Return our choices
          return choices;

        },
        default: function(answers) {
          return answers.database || pantheonConf.env;
        }
      }
    ];
  }
1353796
def _get_top_states(saltenv='base'):
    '''
    
    '''
    alt_states = []
    try:
        returned = __salt__['state.show_top']()
        for i in returned[saltenv]:
            alt_states.append(i)
    except Exception:
        raise
    # log.info("top states: %s", alt_states)
    return alt_states
1293592
def extract_references_from_wets(wet_files, metadata_dir, out_dir,
                                 tmp_dir=None):
  """"""
  # Setup output files
  shard_files = make_ref_shard_files(out_dir)

  num_refs = 0
  for i, wet_file in enumerate(wet_files):
    num_refs_in_wet = 0
    tf.logging.info("Processing file %d", i)

    # Read metadata file
    metadata_fname = os.path.join(
        metadata_dir, os.path.basename(wet_file)) + cc_utils.METADTA_SUFFIX
    with tf.gfile.Open(cc_utils.readahead(metadata_fname)) as f:
      wet_metadata = json.loads(f.read())

    if not wet_metadata:
      # No references in this WET file
      continue

    if wet_file.startswith("http"):
      # download
      if not tmp_dir:
        tmp_dir = tempfile.gettempdir()
      record_gen = cc_utils.wet_records_from_url(wet_file, tmp_dir)
    else:
      # local
      record_gen = cc_utils.wet_records_from_file_obj(
          cc_utils.gzip_memfile(wet_file), take_ownership=True)

    for wet_record in record_gen:
      shard_ids = wet_metadata.get(wet_record.url)
      if not shard_ids:
        # URL not in dataset
        continue

      # Serialize and write out
      ex = _make_example_from_record(wet_record)
      ex_str = ex.SerializeToString()
      for shard_id in shard_ids:
        shard_files[shard_id].write(ex_str)
      num_refs += 1
      num_refs_in_wet += 1

    tf.logging.info("Wrote out %d references for this WET", num_refs_in_wet)

  tf.logging.info("Wrote out %d references total", num_refs)

  # Cleanup
  for shard_file in shard_files:
    shard_file.close()
910034
def file_local_list(self, saltenv='base'):
        '''
        
        '''
        filesdest = os.path.join(self.opts['cachedir'], 'files', saltenv)
        localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles')

        fdest = self._file_local_list(filesdest)
        ldest = self._file_local_list(localfilesdest)
        return sorted(fdest.union(ldest))