Generator function that reads the file index from the vpk file

        yeilds (file_path, metadata)
764051
def read_index_iter(self):
        

        with fopen(self.vpk_path, 'rb') as f:
            f.seek(self.header_length)

            while True:
                if self.version > 0 and f.tell() > self.tree_length + self.header_length:
                    raise ValueError("Error parsing index (out of bounds)")

                ext = _read_cstring(f)
                if ext == '':
                    break

                while True:
                    path = _read_cstring(f)
                    if path == '':
                        break
                    if path != ' ':
                        path = os.path.join(path, '')
                    else:
                        path = ''

                    while True:
                        name = _read_cstring(f)
                        if name == '':
                            break

                        (crc32,
                         preload_length,
                         archive_index,
                         archive_offset,
                         file_length,
                         suffix,
                         ) = metadata = list(struct.unpack("IHHIIH", f.read(18)))

                        if suffix != 0xffff:
                            raise ValueError("Error while parsing index")

                        if archive_index == 0x7fff:
                            metadata[3] = self.header_length + self.tree_length + archive_offset

                        metadata = (f.read(preload_length),) + tuple(metadata[:-1])

                        yield path + name + '.' + ext, metadata
571473
def get_filelist(run_module_names):
    
    # Prep search patterns
    spatterns = [{},{},{},{},{},{},{}]
    epatterns = [{}, {}]
    ignored_patterns = []
    for key, sps in config.sp.items():
        mod_name = key.split('/', 1)[0]
        if mod_name.lower() not in [m.lower() for m in run_module_names]:
            ignored_patterns.append(key)
            continue
        files[key] = list()
        if not isinstance(sps, list):
            sps = [sps]

        # Warn if we have any unrecognised search pattern keys
        expected_sp_keys = [
            'fn',
            'fn_re',
            'contents',
            'contents_re',
            'num_lines',
            'shared',
            'max_filesize',
            'exclude_fn',
            'exclude_fn_re',
            'exclude_contents',
            'exclude_contents_re'
        ]
        unrecognised_keys = [y for x in sps for y in x.keys() if y not in expected_sp_keys]
        if len(unrecognised_keys) > 0:
            logger.warn("Unrecognised search pattern keys for '{}': {}".format(key, ', '.join(unrecognised_keys)))

        # Split search patterns according to speed of execution.
        if any([x for x in sps if 'contents_re' in x]):
            if any([x for x in sps if 'num_lines' in x]):
                spatterns[4][key] = sps
            elif any([x for x in sps if 'max_filesize' in x]):
                spatterns[5][key] = sps
            else:
                spatterns[6][key] = sps
        elif any([x for x in sps if 'contents' in x]):
            if any([x for x in sps if 'num_lines' in x]):
                spatterns[1][key] = sps
            elif any([x for x in sps if 'max_filesize' in x]):
                spatterns[2][key] = sps
            else:
                spatterns[3][key] = sps
        else:
            spatterns[0][key] = sps

    if len(ignored_patterns) > 0:
        logger.debug("Ignored {} search patterns as didn't match running modules.".format(len(ignored_patterns)))

    def add_file(fn, root):
        """
        Function applied to each file found when walking the analysis
        directories. Runs through all search patterns and returns True
        if a match is found.
        """
        f = {'fn': fn, 'root': root}

        # Check that this is a file and not a pipe or anything weird
        if not os.path.isfile(os.path.join(root, fn)):
            return None

        # Check that we don't want to ignore this file
        i_matches = [n for n in config.fn_ignore_files if fnmatch.fnmatch(fn, n)]
        if len(i_matches) > 0:
            logger.debug("Ignoring file as matched an ignore pattern: {}".format(fn))
            return None

        # Limit search to small files, to avoid 30GB FastQ files etc.
        try:
            f['filesize'] = os.path.getsize(os.path.join(root,fn))
        except (IOError, OSError, ValueError, UnicodeDecodeError):
            logger.debug("Couldn't read file when checking filesize: {}".format(fn))
        else:
            if f['filesize'] > config.log_filesize_limit:
                return False

        # Test file for each search pattern
        for patterns in spatterns:
            for key, sps in patterns.items():
                for sp in sps:
                    if search_file (sp, f):
                        # Check that we shouldn't exclude this file
                        if not exclude_file(sp, f):
                            # Looks good! Remember this file
                            files[key].append(f)
                        # Don't keep searching this file for other modules
                        if not sp.get('shared', False):
                            return
                        # Don't look at other patterns for this module
                        else:
                            break

    # Go through the analysis directories and get file list
    for path in config.analysis_dir:
        if os.path.islink(path) and config.ignore_symlinks:
            continue
        elif os.path.isfile(path):
            searchfiles.append([os.path.basename(path), os.path.dirname(path)])
        elif os.path.isdir(path):
            for root, dirnames, filenames in os.walk(path, followlinks=(not config.ignore_symlinks), topdown=True):
                bname = os.path.basename(root)

                # Skip any sub-directories matching ignore params
                orig_dirnames = dirnames[:]
                for n in config.fn_ignore_dirs:
                    dirnames[:] = [d for d in dirnames if not fnmatch.fnmatch(d, n.rstrip(os.sep))]
                    if len(orig_dirnames) != len(dirnames):
                        removed_dirs = [os.path.join(root, d) for d in set(orig_dirnames).symmetric_difference(set(dirnames))]
                        logger.debug("Ignoring directory as matched fn_ignore_dirs: {}".format(", ".join(removed_dirs)))
                        orig_dirnames = dirnames[:]
                for n in config.fn_ignore_paths:
                    dirnames[:] = [d for d in dirnames if not fnmatch.fnmatch(os.path.join(root, d), n.rstrip(os.sep))]
                    if len(orig_dirnames) != len(dirnames):
                        removed_dirs = [os.path.join(root, d) for d in set(orig_dirnames).symmetric_difference(set(dirnames))]
                        logger.debug("Ignoring directory as matched fn_ignore_paths: {}".format(", ".join(removed_dirs)))

                # Skip *this* directory if matches ignore params
                d_matches = [n for n in config.fn_ignore_dirs if fnmatch.fnmatch(bname, n.rstrip(os.sep))]
                if len(d_matches) > 0:
                    logger.debug("Ignoring directory as matched fn_ignore_dirs: {}".format(bname))
                    continue
                p_matches = [n for n in config.fn_ignore_paths if fnmatch.fnmatch(root, n.rstrip(os.sep))]
                if len(p_matches) > 0:
                    logger.debug("Ignoring directory as matched fn_ignore_paths: {}".format(root))
                    continue
                # Search filenames in this directory
                for fn in filenames:
                    searchfiles.append([fn, root])
    # Search through collected files
    with click.progressbar(searchfiles, label="Searching {} files..".format(len(searchfiles))) as sfiles:
        for sf in sfiles:
            add_file(sf[0], sf[1])
979713
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Dictionary of keywords/cards and parse function names
        KEYWORDS = ('NUM_LINKS',
                    'TIME_STEP',
                    'NUM_TS',
                    'START_TIME',
                    'TS')

        # Parse file into chunks associated with keywords/cards
        with open(path, 'r') as f:
            self.name = f.readline().strip()
            chunks = pt.chunk(KEYWORDS, f)

        # Parse chunks associated with each key
        for card, chunkList in iteritems(chunks):
            # Parse each chunk in the chunk list
            for chunk in chunkList:
                schunk = chunk[0].strip().split()

                # Cases
                if card == 'NUM_LINKS':
                    # NUM_LINKS handler
                    self.numLinks = schunk[1]

                elif card == 'TIME_STEP':
                    # TIME_STEP handler
                    self.timeStepInterval = schunk[1]

                elif card == 'NUM_TS':
                    # NUM_TS handler
                    self.numTimeSteps = schunk[1]

                elif card == 'START_TIME':
                    # START_TIME handler
                    self.startTime = '%s  %s    %s  %s  %s  %s' % (
                        schunk[1],
                        schunk[2],
                        schunk[3],
                        schunk[4],
                        schunk[5],
                        schunk[6])

                elif card == 'TS':
                    # TS handler
                    for line in chunk:
                        sline = line.strip().split()
                        token = sline[0]

                        # Cases
                        if token == 'TS':
                            # Time Step line handler
                            timeStep = LinkNodeTimeStep(timeStep=sline[1])
                            timeStep.linkNodeDataset = self

                        else:
                            # Split the line
                            spLinkLine = line.strip().split()

                            # Create LinkDataset GSSHAPY object
                            linkDataset = LinkDataset()
                            linkDataset.numNodeDatasets = int(spLinkLine[0])
                            linkDataset.timeStep = timeStep
                            linkDataset.linkNodeDatasetFile = self

                            # Parse line into NodeDatasets
                            NODE_VALUE_INCREMENT = 2
                            statusIndex = 1
                            valueIndex = statusIndex + 1

                            # Parse line into node datasets
                            if linkDataset.numNodeDatasets > 0:
                                for i in range(0, linkDataset.numNodeDatasets):
                                    # Create NodeDataset GSSHAPY object
                                    nodeDataset = NodeDataset()
                                    nodeDataset.status = int(spLinkLine[statusIndex])
                                    nodeDataset.value = float(spLinkLine[valueIndex])
                                    nodeDataset.linkDataset = linkDataset
                                    nodeDataset.linkNodeDatasetFile = self

                                    # Increment to next status/value pair
                                    statusIndex += NODE_VALUE_INCREMENT
                                    valueIndex += NODE_VALUE_INCREMENT
                            else:
                                nodeDataset = NodeDataset()
                                nodeDataset.value = float(spLinkLine[1])
                                nodeDataset.linkDataset = linkDataset
                                nodeDataset.linkNodeDatasetFile = self
586661
def _find_input_files(var, out):
    
    if isinstance(var, (list, tuple)):
        for x in var:
            out = _find_input_files(x, out)
    elif isinstance(var, dict):
        if var.get("class") == "File":
            out.append(var["path"])
            out = _find_input_files(var.get("secondaryFiles", []), out)
        for key, val in var.items():
            out = _find_input_files(val, out)
    return out
1364895
def get_from_metadata_file(cls, dirpath, key):
        
        fullpath = os.path.join(dirpath, cls.metadata_filename)
        if os.path.exists(fullpath):
            d = pickle.load(open(fullpath, 'rb'))
            return d[key]
        else:
            raise FileNotFoundError(
                "No such file: {0}".format(fullpath))
56043
def get_index(self, filename):
        """"""
        index = self.fsmodel.index(filename)
        if index.isValid() and index.model() is self.fsmodel:
            return self.proxymodel.mapFromSource(index)
763273
def filesfile_string(self):
        """"""
        lines = []
        app = lines.append

        app(self.input_file.path)          # 1) Path of the input file
        app(self.output_file.path)         # 2) Path of the output file
        app(self.ddb_filepath)             # 3) Input derivative database e.g. t13.ddb.in
        app(self.md_filepath)              # 4) Output molecular dynamics e.g. t13.md
        app(self.gkk_filepath)             # 5) Input elphon matrix elements  (GKK file)
        app(self.outdir.path_join("out"))  # 6) Base name for elphon output files e.g. t13
        app(self.ddk_filepath)             # 7) File containing ddk filenames for elphon/transport.

        return "\n".join(lines)
964895
def _file_iter_range(fp, offset, bytes, maxread=1024*1024):
    ''' '''
    fp.seek(offset)
    while bytes > 0:
        part = fp.read(min(bytes, maxread))
        if not part: break
        bytes -= len(part)
        yield part
935794
function createIndex() {
  var directory = 'src/views'
  var directories = fs.readdirSync(directory)

  try {
    var lines = ['// This is a generated file, do not edit, or disable "prebuild" command in package.json if you want to take control']
    for (var i = 0; i < directories.length; i++) {
      var path = directory + '/' + directories[i];
      if (fs.existsSync(path) && fs.lstatSync(path).isDirectory()) {
        var file = directories[i] + '.js'
        if (fs.existsSync(directory + '/' + directories[i] + '/' + file)) {
          lines.push('export { default as ' + directories[i] + ' } from \'./' + directories[i] + '/' + directories[i] + '\'')
        }
      }
    }

    fs.writeFileSync(directory + '/index.js', lines.join('\n') + '\n');
  } catch (err) {
    console.log(err)
  }
}
148214
def read_file(input_filename):
        ''''''
        with open(input_filename) as f:
            g = GreenGenesTaxonomy.read(f)
        return g
196925
def file_names(self) -> Iterable[str]:
        
        if self.is_sharded():
            yield from ShardedFile(self.filename_spec).get_filenames()
        elif self.filename_spec:
            yield self.filename_spec
354556
def get_index_from_filename(self, filename):
        
        filenames = [d.filename for d in self.data]
        return filenames.index(filename)
1049307
def read_files(*sources, **kwds):
    
    filenames = _generate_filenames(sources)
    filehandles = _generate_handles(filenames)
    for fh, source in filehandles:
        try:
            f = mwtab.MWTabFile(source)
            f.read(fh)

            if kwds.get('validate'):
                validator.validate_file(mwtabfile=f,
                                        section_schema_mapping=mwschema.section_schema_mapping,
                                        validate_samples=True,
                                        validate_factors=True)
            yield f

            if VERBOSE:
                print("Processed file: {}".format(os.path.abspath(source)))

        except Exception as e:
            if VERBOSE:
                print("Error processing file: ", os.path.abspath(source), "\nReason:", e)
            pass
381056
def read_index(group, version='1.1'):
    
    if version == '0.1':
        return np.int64(group['index'][...])
    elif version == '1.0':
        return group['file_index'][...]
    else:
        return group['index'][...]
1015131
def _load(formula):
    '''
    
    '''

    # Compute possibilities
    _mk_client()
    paths = []
    for ext in ('yaml', 'json'):
        source_url = salt.utils.url.create(formula + '/defaults.' + ext)
        paths.append(source_url)
    # Fetch files from master
    defaults_files = __context__['cp.fileclient'].cache_files(paths)

    for file_ in defaults_files:
        if not file_:
            # Skip empty string returned by cp.fileclient.cache_files.
            continue

        suffix = file_.rsplit('.', 1)[-1]
        if suffix == 'yaml':
            loader = salt.utils.yaml.safe_load
        elif suffix == 'json':
            loader = salt.utils.json.load
        else:
            log.debug("Failed to determine loader for %r", file_)
            continue

        if os.path.exists(file_):
            log.debug("Reading defaults from %r", file_)
            with salt.utils.files.fopen(file_) as fhr:
                defaults = loader(fhr)
                log.debug("Read defaults %r", defaults)

            return defaults or {}
489541
def get_file(self, path):
        
        metadata = self.get_file_meta(path)
        return self.get_vpkfile_instance(path, metadata)
643239
def read_file(file_path):
        
        with open(file_path, 'r', encoding='utf-8') as f:
            whole = f.read().strip()

        if whole.startswith('---'):
            # may has yaml meta info, so we try to split it out
            sp = re.split(r'-{3,}', whole.lstrip('-'), maxsplit=1)
            if len(sp) == 2:
                # do have yaml meta info, so we read it
                return yaml.load(sp[0]), sp[1].lstrip()
        return {}, whole