Locally cache a remote resource using a predictable file name
        and awareness of modification date. Assume that files are "normal"
        which is to say they have filenames with extensions.
775159
def locally_cache_remote_file(href, dir):
    
    scheme, host, remote_path, params, query, fragment = urlparse(href)
    
    assert scheme in ('http','https'), 'Scheme must be either http or https, not "%s" (for %s)' % (scheme,href)

    head, ext = posixpath.splitext(posixpath.basename(remote_path))
    head = sub(r'[^\w\-_]', '', head)
    hash = md5(href).hexdigest()[:8]
    
    local_path = '%(dir)s/%(host)s-%(hash)s-%(head)s%(ext)s' % locals()

    headers = {}
    if posixpath.exists(local_path):
        msg('Found local file: %s' % local_path )
        t = localtime(os.stat(local_path).st_mtime)
        headers['If-Modified-Since'] = strftime('%a, %d %b %Y %H:%M:%S %Z', t)
    
    if scheme == 'https':
        conn = HTTPSConnection(host, timeout=5)
    else:
        conn = HTTPConnection(host, timeout=5)

    if query:
        remote_path += '?%s' % query

    conn.request('GET', remote_path, headers=headers)
    resp = conn.getresponse()
        
    if resp.status in range(200, 210):
        # hurrah, it worked
        f = open(un_posix(local_path), 'wb')
        msg('Reading from remote: %s' % remote_path)
        f.write(resp.read())
        f.close()

    elif resp.status in (301, 302, 303) and resp.getheader('location', False):
        # follow a redirect, totally untested.
        redirected_href = urljoin(href, resp.getheader('location'))
        redirected_path = locally_cache_remote_file(redirected_href, dir)
        os.rename(redirected_path, local_path)
    
    elif resp.status == 304:
        # hurrah, it's cached
        msg('Reading directly from local cache')
        pass

    else:
        raise Exception("Failed to get remote resource %s: %s" % (href, resp.status))
    
    return local_path
546766
def _add_remote_resources(resources):
    
    out = copy.deepcopy(resources)
    for prog, info in resources.items():
        for key, val in info.items():
            if key == "jar" and objectstore.is_remote(val):
                store_dir = utils.safe_makedir(os.path.join(os.getcwd(), "inputs", "jars", prog))
                fname = objectstore.download(val, store_dir, store_dir)
                version_file = os.path.join(store_dir, "version.txt")
                if not utils.file_exists(version_file):
                    version = install.get_gatk_jar_version(prog, fname)
                    with open(version_file, "w") as out_handle:
                        out_handle.write(version)
                else:
                    with open(version_file) as in_handle:
                        version = in_handle.read().strip()
                del out[prog][key]
                out[prog]["dir"] = store_dir
                out[prog]["version"] = version
    return out
1126118
def cache_dest(self, url, saltenv='base', cachedir=None):
        '''
        
        '''
        proto = urlparse(url).scheme

        if proto == '':
            # Local file path
            return url

        if proto == 'salt':
            url, senv = salt.utils.url.parse(url)
            if senv:
                saltenv = senv
            return salt.utils.path.join(
                self.opts['cachedir'],
                'files',
                saltenv,
                url.lstrip('|/'))

        return self._extrn_path(url, saltenv, cachedir=cachedir)
589077
def should_copy_file(remote_storage, path, prefixed_path, source_storage):
    
    if has_matching_etag(
            remote_storage, source_storage, path, prefixed_path):
        logger.info("%s: Skipping based on matching file hashes" % path)
        return False

    # Invalidate cached versions of lookup before copy
    destroy_etag(path)
    logger.info("%s: Hashes did not match" % path)
    return True
1172688
def cache_path(archive, root_dir, build_id):
    
    root = root_dir or Path("~/.shiv").expanduser()
    name = Path(archive.filename).resolve().stem
    return root / f"{name}_{build_id}"
1104766
def _create_cache_filename(self, source_file):
        
        res = self._create_cache_key(source_file) + ".cache"
        return os.path.join(self.__dir, res)
460085
def set(self, prefix, url, obj):
        
        if not self.cache_dir:
            return

        filename = self._get_cache_file(prefix, url)

        try:
            os.makedirs(os.path.join(self.cache_dir, prefix))
        except OSError:
            pass

        with open(filename, 'wb') as file:
            pickle.dump(obj, file)
1291494
def _get_cache(vcf_fn, array_type, region, cachedir, compress, log):
    """"""

    # guard condition
    if isinstance(vcf_fn, (list, tuple)):
        raise Exception(
            'caching only supported when loading from a single VCF file'
        )

    # create cache file name
    cache_fn = _mk_cache_fn(vcf_fn, array_type=array_type, region=region,
                            cachedir=cachedir, compress=compress)

    # decide whether or not a fresh cache file is available
    # (if not, we will parse the VCF and build array from scratch)
    if not os.path.exists(cache_fn):
        log('no cache file found')
        is_cached = False
    elif os.path.getmtime(vcf_fn) > os.path.getmtime(cache_fn):
        is_cached = False
        log('cache file out of date')
    else:
        is_cached = True
        log('cache file available')

    return cache_fn, is_cached
531773
def _load_cache(self, filename):
        """"""
        try:
            with open(filename + self.CACHE_EXTENSION, 'rb') as file:
                prev_number_of_pages, prev_page_references = pickle.load(file)
        except (IOError, TypeError):
            prev_number_of_pages, prev_page_references = {}, {}
        return prev_number_of_pages, prev_page_references
247002
def hashed(source_filename, prepared_options, thumbnail_extension, **kwargs):
    
    parts = ':'.join([source_filename] + prepared_options)
    short_sha = hashlib.sha1(parts.encode('utf-8')).digest()
    short_hash = base64.urlsafe_b64encode(short_sha[:9]).decode('utf-8')
    return '.'.join([short_hash, thumbnail_extension])
1335724
def glob_files_remotely(cmd, client, share_name, pattern):
    """"""
    from collections import deque
    t_dir, t_file = cmd.get_models('file.models#Directory', 'file.models#File')

    queue = deque([""])
    while queue:
        current_dir = queue.pop()
        for f in client.list_directories_and_files(share_name, current_dir):
            if isinstance(f, t_file):
                if not pattern or _match_path(os.path.join(current_dir, f.name), pattern):
                    yield current_dir, f.name
            elif isinstance(f, t_dir):
                queue.appendleft(os.path.join(current_dir, f.name))
1306550
def cache_file(self, path, saltenv='base', cachedir=None, source_hash=None):
        '''
        
        '''
        return self.get_url(
            path, '', True, saltenv, cachedir=cachedir, source_hash=source_hash)
660504
function ensureFile(filename) {
    var base = path.dirname(filename);
    return Promise.nfcall(mkdirp, base);
}
190914
def local_cache_path(remote_url):
    """"""
    local_name = RESERVED_PATH_CHARS.sub("_", remote_url)
    return os.path.join(gettempdir(), local_name)
448696
def _get_cache_filename(name, filename):
    """"""
    filename = os.path.abspath(filename)[1:]
    home_folder = os.path.expanduser('~')
    base_cache_dir = os.path.join(home_folder, '.git-lint', 'cache')

    return os.path.join(base_cache_dir, name, filename)
222874
def get_fileinfo(filespec, cache_dir=None):
    
    if cache_dir is None:
        cache_dir = tempfile.gettempdir()

    # Loads first science extension by default.
    # This prevents [None] to be loaded instead.
    idx = None
    name_ext = ''

    # User specified an index using bracket notation at end of path?
    match = re.match(r'^(.+)\[(.+)\]$', filespec)
    if match:
        filespec = match.group(1)
        idx = match.group(2)
        if ',' in idx:
            hduname, extver = idx.split(',')
            hduname = hduname.strip()
            if re.match(r'^\d+$', extver):
                extver = int(extver)
                idx = (hduname, extver)
                name_ext = "[%s,%d]" % idx
            else:
                idx = idx.strip()
                name_ext = "[%s]" % idx
        else:
            if re.match(r'^\d+$', idx):
                idx = int(idx)
                name_ext = "[%d]" % idx
            else:
                idx = idx.strip()
                name_ext = "[%s]" % idx
    else:
        filespec = filespec

    url = filespec
    filepath = None

    # Does this look like a URL?
    match = re.match(r"^(\w+)://(.+)$", filespec)
    if match:
        urlinfo = urllib.parse.urlparse(filespec)
        if urlinfo.scheme == 'file':
            # local file
            filepath = str(pathlib.Path(urlinfo.path))

        else:
            path, filename = os.path.split(urlinfo.path)
            filepath = get_download_path(url, filename, cache_dir)

    else:
        # Not a URL
        filepath = filespec
        url = pathlib.Path(os.path.abspath(filepath)).as_uri()

    ondisk = os.path.exists(filepath)

    dirname, fname = os.path.split(filepath)
    fname_pfx, fname_sfx = os.path.splitext(fname)
    name = fname_pfx + name_ext

    res = Bunch.Bunch(filepath=filepath, url=url, numhdu=idx,
                      name=name, idx=name_ext, ondisk=ondisk)
    return res
228476
def cache_set(cache_dir, cache_key, content):
    
    filename = os.path.join(cache_dir, cache_key)

    with open(filename, 'w') as f:
        f.write(content)