Return download data by file type.

        :return: dict of cache data; keys are datetime objects, values are
          dict of file type (str) to count (int)
        :rtype: dict
519180
def per_file_type_data(self):
        
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            if len(data['by_file_type']) == 0:
                data['by_file_type'] = {'other': 0}
            ret[cache_date] = data['by_file_type']
        return ret
1195059
def _file_timestamp_info(self, path):
        
        files = [os.path.join(path, fname) for fname in os.listdir(path) if '.py' in fname]
        return dict ([(fname, os.path.getmtime(fname)) for fname in files])
63428
def generate_and_cache(path=user_path):
    '''
    
    '''
    data = tools.generate()
    if not path:
        return data
    try:
        directory = os.path.dirname(path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        with open(path, 'wb') as f:
            pickle.dump((data_version, module_version, data), f)
    except (PermissionError, ValueError) as e:
        warnings.warn('Unable to write cache file "%s": %s' % (path, e))
    return data
1020567
def read(filename='cache'):
	
	cache_path = get_cache_path(filename)
	if not os.path.exists(cache_path) or os.stat(cache_path).st_size == 0:
		return None
	with open(cache_path, 'r') as file:
		return json.load(file)
1306550
def cache_file(self, path, saltenv='base', cachedir=None, source_hash=None):
        '''
        
        '''
        return self.get_url(
            path, '', True, saltenv, cachedir=cachedir, source_hash=source_hash)
22260
def download_files(fapi, file_name, conf, use_cache, cache_dir=None):
    
    retrieval_type              = conf.get('retrieval-type', 'published')
    include_original_strings    = 'true' if conf.get('include-original-strings', False) else 'false'
    save_pattern                = conf.get('save-pattern')

    if not save_pattern:
        raise SmarterlingError("File %s doesn't have a save-pattern" % file_name)

    if cache_dir and not os.path.exists(cache_dir):
        print("Creating cache dir: %s" % (cache_dir, ))
        os.makedirs(cache_dir)

    uri = file_uri(file_name, conf)
    translated_items = get_translated_items(fapi, uri, use_cache, cache_dir=cache_dir)

    for item in translated_items:
        item                = AttributeDict(item)
        locale              = item.locale
        locale_underscore   = locale.replace("-", "_")
        locale_android_res  = locale.replace("-", "-r")
        locale_parts        = locale.split("-")
        language            = locale_parts[0]
        region              = locale_parts[1] if len(locale_parts)>1 else ""

        file_response = get_translated_file(
            fapi, file_uri(file_name, conf), locale, retrieval_type,
            include_original_strings, use_cache, cache_dir=cache_dir)

        if not file_response:
            print("%s translation not found for %s" % (item.locale, file_name))
            continue
        print("Processing %s translation for %s" % (item.locale, file_name))

        (fd, work_file) = tempfile.mkstemp()
        try:
            with open(work_file, 'w') as f:
                f.write(file_response)
        finally:
            os.close(fd)

        for filter_cmd in conf.get('filters', []):
            (fd, tmp_file) = tempfile.mkstemp()
            try :
                filter_cmd = filter_cmd.replace("{input_file}", work_file)
                filter_cmd = filter_cmd.replace("{output_file}", tmp_file)
                filter_cmd = filter_cmd.replace("{locale}", locale)
                filter_cmd = filter_cmd.replace("{locale_underscore}", locale_underscore)
                filter_cmd = filter_cmd.replace("{locale_android_res}", locale_android_res)
                filter_cmd = filter_cmd.replace("{language}", language)
                filter_cmd = filter_cmd.replace("{region}", region)
                print(" running filter: %s " % filter_cmd)
                if os.system(filter_cmd) != 0:
                    raise SmarterlingError("Non 0 exit code from filter: %s" % filter_cmd)
                shutil.move(tmp_file, work_file)
            finally:
                os.close(fd)

        if conf.has_key('save-cmd'):
            save_command = conf.get('save-cmd')
            save_command = save_command.replace("{input_file}", work_file)
            save_command = save_command.replace("{locale}", locale)
            save_command = save_command.replace("{locale_underscore}", locale_underscore)
            save_command = save_command.replace("{locale_android_res}", locale_android_res)
            save_command = save_command.replace("{language}", language)
            save_command = save_command.replace("{region}", region)
            print(" running save command: %s " % save_command)
            if os.system(save_command) != 0:
                raise SmarterlingError("Non 0 exit code from save command: %s" % save_command)

        elif conf.has_key('save-pattern'):
            save_file = conf.get('save-pattern')
            save_file = save_file.replace("{locale}", locale)
            save_file = save_file.replace("{locale_underscore}", locale_underscore)
            save_file = save_file.replace("{locale_android_res}", locale_android_res)
            save_file = save_file.replace("{language}", language)
            save_file = save_file.replace("{region}", region)
            save_dir = os.path.dirname(save_file)
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            elif not os.path.isdir(save_dir):
                raise SmarterlingError("Expected %s to be a directory, but it's an existing file" % save_dir)
            print(" saving output to: %s " % save_file)
            shutil.move(work_file, save_file)

        else:
            raise SmarterlingError("no save-cmd or save-pattern for: %s" % file_name)
469165
def get_cached_files(url, server_name="", document_root=None):
    
    import glob
    url_info = process_url(url, server_name, document_root, check_security=False)

    # get path to cache directory with basename of file (no extension)
    filedir = os.path.dirname(url_info['requested_file'])
    fileglob = '{0}*{1}'.format(url_info['base_filename'], url_info['ext'])
    return glob.glob(os.path.join(filedir, fileglob))
141855
def read_file(filename: str, force_reload=False):
    

    if force_reload:
        _delete_cached_file(filename)

    reloaded = False
    mtime = os.path.getmtime(filename)
    cache_info = CACHE.setdefault(filename, {})

    if not cache_info or mtime > cache_info.get('mtime', 0):
        LOG.debug('Reloading cached file %s', filename)
        with open(filename) as fp:
            cache_info['data'] = fp.read()
        cache_info['mtime'] = mtime
        reloaded = True

    return reloaded, cache_info['data']
1022085
def to_api_data(self):
        
        data = {'@odata.type': self._gk(
            '{}_attachment_type'.format(self.attachment_type)),
            self._cc('name'): self.name}

        if self.attachment_type == 'file':
            data[self._cc('contentBytes')] = self.content
        else:
            data[self._cc('item')] = self.content

        return data
1207239
def to_dict(self):
        
        data = dict()
        data['max_age'] = self.max_age
        data['cache_dir'] = self.cache_dir
        return data
424994
def getDownload(self, params):
        
        df = self.getData(params)
        buffer = io.StringIO()
        df.to_csv(buffer, index=False, encoding='utf-8')
        filepath = buffer
        return filepath
1005085
async def async_get_bytes_total(self, use_cache=True):
        """"""
        now = datetime.utcnow()
        if use_cache and self._trans_cache_timer and self._cache_time > \
                (now - self._trans_cache_timer).total_seconds():
            return self._transfer_rates_cache

        rx = await self.async_get_rx()
        tx = await self.async_get_tx()
        return rx, tx
1310784
def get_from_cache(url: str, cache_dir: str = None) -> str:
    
    if cache_dir is None:
        cache_dir = CACHE_DIRECTORY

    os.makedirs(cache_dir, exist_ok=True)

    # Get eTag to add to filename, if it exists.
    if url.startswith("s3://"):
        etag = s3_etag(url)
    else:
        response = requests.head(url, allow_redirects=True)
        if response.status_code != 200:
            raise IOError("HEAD request failed for url {} with status code {}"
                          .format(url, response.status_code))
        etag = response.headers.get("ETag")

    filename = url_to_filename(url, etag)

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    if not os.path.exists(cache_path):
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:
            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)

            # GET file object
            if url.startswith("s3://"):
                s3_get(url, temp_file)
            else:
                http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
            with open(cache_path, 'wb') as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            logger.info("creating metadata file for %s", cache_path)
            meta = {'url': url, 'etag': etag}
            meta_path = cache_path + '.json'
            with open(meta_path, 'w') as meta_file:
                json.dump(meta, meta_file)

            logger.info("removing temp file %s", temp_file.name)

    return cache_path
896598
def get_files_information(self):
        
        if self.files == {}:
            self.get_files_types()

        for i in self.get_files():
            try:
                yield i, self.files[i], self.files_crc32[i]
            except KeyError:
                yield i, "", ""
292653
def cache_info(cache_dir, cache_key):
    
    filename = os.path.join(cache_dir, cache_key)
    return os.path.getmtime(filename) if os.path.exists(filename) else 0
331600
def cached_read(self, kind):
        """"""
        if not kind in self.cache:
            self.pull_stats(kind)
        if self.epochnow() - self.cache[kind]['lastcall'] > self.cache_timeout:
            self.pull_stats(kind)
        return self.cache[kind]['lastvalue']
48472
def get(self, attach, *args, **kwargs):
        
        response = self.make_response(*args, **kwargs)  # type: Response
        response.content_type = self.get_content_type(*args, **kwargs)

        if attach:
            filename = self.get_filename(*args, **kwargs)
            if not filename:
                filename = "file.bin"
            headers = response.headers
            headers.add("Content-Disposition", "attachment", filename=filename)

        self.set_cache_headers(response)
        return response