Fetches the given URLs and caches their contents
        and their assets in the given directory.
4117
def _cache_contents(self, style_urls, asset_url_path):
        
        files = {}

        asset_urls = []
        for style_url in style_urls:
            if not self.quiet:
                print(' * Downloading style', style_url, file=sys.stderr)
            r = requests.get(style_url)
            if not 200 <= r.status_code < 300:
                print(' -> Warning: Style request responded with',
                      r.status_code, file=sys.stderr)
                files = None
                continue
            asset_content = r.text
            # Find assets and replace their base URLs with the cache directory
            for url in re.findall(STYLE_ASSET_URLS_RE, asset_content):
                asset_urls.append(urljoin(style_url, url))
            contents = re.sub(
                STYLE_ASSET_URLS_RE,
                STYLE_ASSET_URLS_SUB_FORMAT.format(asset_url_path.rstrip('/')),
                asset_content)
            # Prepare cache
            if files is not None:
                filename = self.cache_filename(style_url)
                files[filename] = contents.encode('utf-8')

        for asset_url in asset_urls:
            if not self.quiet:
                print(' * Downloading asset', asset_url, file=sys.stderr)
            # Retrieve binary file and show message
            r = requests.get(asset_url, stream=True)
            if not 200 <= r.status_code < 300:
                print(' -> Warning: Asset request responded with',
                      r.status_code, file=sys.stderr)
                files = None
                continue
            # Prepare cache
            if files is not None:
                filename = self.cache_filename(asset_url)
                files[filename] = r.raw.read(decode_content=True)

        # Skip caching if something went wrong to try again next time
        if not files:
            return False

        # Cache files if all downloads were successful
        cache = {}
        for relname in files:
            cache[safe_join(self.cache_path, relname)] = files[relname]
        if not os.path.exists(self.cache_path):
            os.makedirs(self.cache_path)
        for filename in cache:
            with open(filename, 'wb') as f:
                f.write(cache[filename])
        if not self.quiet:
            print(
                ' * Cached all downloads in', self.cache_path, file=sys.stderr)
        return True
1286605
def fetch(self, url, encoding=None, force_refetch=False, nocache=False, quiet=True):
        ''' '''
        try:
            if not force_refetch and self.cache is not None and url in self.cache:
                # try to look for content in cache
                logging.debug('Retrieving content from cache for {}'.format(url))
                return self.cache.retrieve_blob(url, encoding)
            encoded_url = WebHelper.encode_url(url)
            req = Request(encoded_url, headers={'User-Agent': 'Mozilla/5.0'})
            # support gzip
            req.add_header('Accept-encoding', 'gzip, deflate')
            # Open URL
            getLogger().info("Fetching: {url} |".format(url=url))
            response = urlopen(req)
            content = response.read()
            # unzip if required
            if 'Content-Encoding' in response.info() and response.info().get('Content-Encoding') == 'gzip':
                # unzip
                with gzip.open(BytesIO(content)) as gzfile:
                    content = gzfile.read()
            # update cache if required
            if self.cache is not None and not nocache:
                if url not in self.cache:
                    self.cache.insert_blob(url, content)
            return content.decode(encoding) if content and encoding else content
        except URLError as e:
            if hasattr(e, 'reason'):
                getLogger().exception('We failed to reach {}. Reason: {}'.format(url, e.reason))
            elif hasattr(e, 'code'):
                getLogger().exception('The server couldn\'t fulfill the request. Error code: {}'.format(e.code))
            else:
                # Other exception ...
                getLogger().exception("Fetching error")
            if not quiet:
                raise
        return None
754110
public function loadRoutes()
	{
		if ($this->bound('router')) {
			/**
			 * Use Cached Routes if Available
			 */

			if ($this['config']->get('routes.cache') && $this['files']->exists($this['config']->get('routes.compiled'))) {
				$contents = $this['files']->get($this['config']->get('routes.compiled'));
				if (!empty($contents)) {
					//Get Cached Routes & Set Them
					$this['router']->setRoutes(unserialize(base64_decode($contents)));
				}
			} else {
				//Assign Router to Simple Variable for Include
				$route = $this['router'];
				$response = $this['response'];
				//Include Routes
				if($this['files']->exists($this->base_path . '/app/Http/routes.php')){

					require_once $this->base_path . '/app/Http/routes.php';
				}
			}
			/**
			 * Store Cached Routes
			 */
			if ($this['config']->get('routes.cache') && !$this['files']->exists($this['config']->get('routes.compiled'))) {
				try {
					//Set Routes Cache
					if (!$this['files']->exists($this['config']->get('routes.compiled'))) {
						$allRoutes = $this['router']->getRoutes();
						//If Routes then Serialize
						if (count($allRoutes) > 0) {
							foreach ($allRoutes as $routeObject) {
								$routeObject->prepareForSerialization();
							}
						}
						//Store Routes in Cache
						$this['files']->put($this['config']->get('routes.compiled'), base64_encode(serialize($allRoutes)));
					}
				} catch (\Exception $exception) {
					if (!empty($exception->getMessage())) {
						add_action('admin_notices', function () use ($exception) {
							?>
							<div class="error notice">
								<p>&#9888; <?php echo $exception->getMessage(); ?></p>
								<p><em><? echo 'Route caching cannot serialize closures'; ?>.</em></p>
							</div>
							<?php
						});
					}
				}
			}
		}
	}
318442
def _fetch_parent_path(self, request: Request, use_cache: bool=True):
        '''
        '''
        directory_url = to_dir_path_url(request.url_info)

        if use_cache:
            if directory_url in self._processor.listing_cache:
                return self._processor.listing_cache[directory_url]

        directory_request = copy.deepcopy(request)
        directory_request.url = directory_url

        _logger.debug('Check if URL {} is file with {}.', request.url,
                      directory_url)

        with self._processor.ftp_client.session() as session:
            try:
                yield from session.start_listing(directory_request)
            except FTPServerError:
                _logger.debug('Got an error. Assume is file.')

                if use_cache:
                    self._processor.listing_cache[directory_url] = None

                return

            temp_file = tempfile.NamedTemporaryFile(
                dir=self._item_session.app_session.root_path,
                prefix='tmp-wpull-list'
            )

            with temp_file as file:
                directory_response = yield from session.download_listing(
                    file, duration_timeout=self._fetch_rule.duration_timeout)

        if use_cache:
            self._processor.listing_cache[directory_url] = \
                directory_response.files

        return directory_response.files
470121
def _get_styles(self, style_urls, asset_url_path):
        
        styles = []
        for style_url in style_urls:
            urls_inline = STYLE_ASSET_URLS_INLINE_FORMAT.format(
                asset_url_path.rstrip('/'))
            asset_content = self._download(style_url)
            content = re.sub(urls_inline, self._match_asset, asset_content)
            styles.append(content)

        return styles
1162202
function loadResources() {
        var resources = {};
        var resourcesDir = path.join(this.injector.rootDir, this.injector.servicesDir + '/resources');


        if (!fs.existsSync(resourcesDir)) {
            return resources;
        }

        var me = this;
        var resourceNames = fs.readdirSync(resourcesDir);

        _.each(resourceNames, function (resourceName) {
            if (resourceName.substring(resourceName.length - 3) !== '.js') {  // only dirs, not js files
                resources[resourceName] = me.injector.loadModule(utils.getCamelCase(resourceName + '.resource'));
            }
        });

        return resources;
    }
400980
protected function cacheMediaAssetFiles()
    {
        $aAssetFileFilters = array();
        $aAssetFiles = array();

        //Retrieve asset files cache manager
        $oAssetFilesCacheManager = $this->getAssetFilesCacheManager();

        // Retrieve asset file filters manager
        $oAssetFileFiltersManager = $this->getAssetFileFiltersManager();
        $bIsProduction = $this->getOptions()->isProduction();

        foreach ($this->getAssetFilesConfiguration()->getAssetFiles(\AssetsBundle\AssetFile\AssetFile::ASSET_MEDIA) as $oAssetFile) {
            if ($oAssetFilesCacheManager->isAssetFileCached($oAssetFile)) {
                $aAssetFiles[] = $oAssetFilesCacheManager->getAssetFileCachePath($oAssetFile);
                continue;
            }
            if ($bIsProduction) {
                if (array_key_exists($sAssetFileExtension = $oAssetFile->getAssetFileExtension(), $aAssetFileFilters)) {
                    $oMediaFileFilter = $aAssetFileFilters[$sAssetFileExtension];
                } else {
                    $oMediaFileFilter = $aAssetFileFilters[$sAssetFileExtension] = $oAssetFileFiltersManager->has($sAssetFileExtension) ? $oAssetFileFiltersManager->get($sAssetFileExtension) : null;
                }
            } else {
                $oMediaFileFilter = null;
            }

            if ($oMediaFileFilter) {
                $oTmpAssetFile = $this->createTmpAssetFile(\AssetsBundle\AssetFile\AssetFile::ASSET_MEDIA);
                $oTmpAssetFile->setAssetFileContents($oMediaFileFilter->filterAssetFile($oAssetFile), false);

                // Cache asset file
                $aAssetFiles[] = $oAssetFilesCacheManager->cacheAssetFile($oTmpAssetFile, $oAssetFile);
            } else {
                // Cache asset file
                $aAssetFiles[] = $oAssetFilesCacheManager->cacheAssetFile($oAssetFile);
            }
        }

        return $aAssetFiles;
    }
1093938
def _load(self, url):
        
        self._logger.debug('Loading url %s into resource cache' % url)
        retriever = get_resource_retriever(url)
        content_path = os.path.join(self.path, self._hash_path(url))

        try:
            if url in self.metadata:
                headers = retriever.fetch(content_path, last_etag=self.metadata[url]['etag'], progress_bar=self.progress_bars)
                if headers is None:
                    # no update, return what is already loaded
                    self._logger.info('Cached %s is up-to-date. No data download needed' % url)
                    return self.metadata[url]
            else:
                headers = retriever.fetch(content_path, progress_bar=self.progress_bars)
                if headers is None:
                    raise Exception('Fetch of %s failed' % url)

            if 'etag' not in headers:
                # No Etag returned, so generate it
                headers['etag'] = fs_util.calc_file_md5(content_path)

            # Populate metadata from the headers
            self.metadata[url] = headers.copy()
            self.metadata[url]['content'] = content_path
            return self.metadata[url]
        except:
            self._logger.error('Failed getting resource: %s')

            # forcibly flush local entry if found
            if url in self.metadata:
                self.metadata.pop(url)
            raise
        finally:
            if url not in self.metadata:
                self._logger.debug('Cleaning up on failed load of %s' % url)
                # Cleanup on failure
                if content_path is not None and os.path.exists(content_path):
                    os.remove(content_path)
1111349
private Set<URL> filterURLs(final Set<URL> urls) {
		final Set<URL> results = new HashSet<URL>(urls.size());
		for (final URL url : urls) {
			String cleanURL = url.toString();
			// Fix JBoss URLs
			if (url.getProtocol().startsWith("vfszip:")) {
				cleanURL = cleanURL.replaceFirst("vfszip:", "file:");
			} else if (url.getProtocol().startsWith("vfsfile:")) {
				cleanURL = cleanURL.replaceFirst("vfsfile:", "file:");
			}
			cleanURL = cleanURL.replaceFirst("\\.jar/", ".jar!/");
			try {
				results.add(new URL(cleanURL));
			} catch (final MalformedURLException ex) {
				// Shouldn't happen, but we can't do more to fix this URL.
			}
		}
		return results;
	}
305429
def load_cache
      begin
        file = File.join("inline", File.basename(so_name))
        if require file then
          dir = Inline.directory
          warn "WAR\NING: #{dir} exists but is not being used" if test ?d, dir and $VERBOSE
          return true
        end
      rescue LoadError
      end
      return false
    end
791670
def urlretrieve(uri, saveas=None, retries=3, cache_dir=None):
    ''''''
    retries = int(retries) if retries else 3
    # FIXME: make random filename (saveas) in cache_dir...
    # cache_dir = cache_dir or CACHE_DIR
    while retries:
        try:
            _path, headers = urllib.urlretrieve(uri, saveas)
        except Exception as e:
            retries -= 1
            logger.warn(
                'Failed getting uri "%s": %s (retry:%s in 1s)' % (
                    uri, e, retries))
            time.sleep(.2)
            continue
        else:
            break
    else:
        raise RuntimeError("Failed to retrieve uri: %s" % uri)
    return _path
1294044
public static URL[] getURLs(File... files) {
		final URL[] urls = new URL[files.length];
		try {
			for (int i = 0; i < files.length; i++) {
				urls[i] = files[i].toURI().toURL();
			}
		} catch (MalformedURLException e) {
			throw new UtilException(e, "Error occured when get URL!");
		}

		return urls;
	}
437428
function getResources( filePath ) {
	if ( fs.existsSync( filePath ) ) {
		return readDirectory( filePath )
			.then( function( contents ) {
				return _.map( contents, function( item ) {
					var resourcePath = path.join( filePath, item, 'resource.js' );
					if ( fs.existsSync( resourcePath ) ) {
						return resourcePath;
					}
				}.bind( this ) );
			}.bind( this ) );
	} else {
		return when( [] );
	}
}
1357413
def fetch_urls(urls, timeout=None):
        
        threads = [threading.Thread(target=SimpleCrawler._fetch_url, args=(url, True, timeout)) for url in urls]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()

        results = copy.deepcopy(SimpleCrawler._results)
        SimpleCrawler._results = {}
        return results
232241
def download_urls(self):
    """"""
    urls = {
        tfds.Split.TRAIN: ["train_clean100"],
        tfds.Split.VALIDATION: ["dev_clean"],
        tfds.Split.TEST: ["test_clean"],
    }
    if self.data in ["all", "clean360"]:
      urls[tfds.Split.TRAIN].append("train_clean360")
    if self.data == "all":
      urls[tfds.Split.TRAIN].extend(["train_clean360", "train_other500"])
      urls[tfds.Split.VALIDATION].append("dev_other")
      urls[tfds.Split.TEST].append("test_other")

    urls = {
        split: [_DL_URLS[name] for name in names
               ] for split, names in urls.items()
    }
    return urls
525277
def load(uri)
      unloaded = UnloadedAsset.new(uri, self)
      if unloaded.params.key?(:id)
        unless asset = asset_from_cache(unloaded.asset_key)
          id = unloaded.params.delete(:id)
          uri_without_id = build_asset_uri(unloaded.filename, unloaded.params)
          asset = load_from_unloaded(UnloadedAsset.new(uri_without_id, self))
          if asset[:id] != id
            @logger.warn "Sprockets load error: Tried to find #{uri}, but latest was id #{asset[:id]}"
          end
        end
      else
        asset = fetch_asset_from_dependency_cache(unloaded) do |paths|
          # When asset is previously generated, its "dependencies" are stored in the cache.
          # The presence of `paths` indicates dependencies were stored.
          # We can check to see if the dependencies have not changed by "resolving" them and
          # generating a digest key from the resolved entries. If this digest key has not
          # changed, the asset will be pulled from cache.
          #
          # If this `paths` is present but the cache returns nothing then `fetch_asset_from_dependency_cache`
          # will confusingly be called again with `paths` set to nil where the asset will be
          # loaded from disk.
          if paths
            digest = DigestUtils.digest(resolve_dependencies(paths))
            if uri_from_cache = cache.get(unloaded.digest_key(digest), true)
              asset_from_cache(UnloadedAsset.new(uri_from_cache, self).asset_key)
            end
          else
            load_from_unloaded(unloaded)
          end
        end
      end
      Asset.new(asset)
    end
56816
private void loadFileSources(final File directory, final String project) {
        //clear source sequence

        if (!directory.isDirectory()) {
            logger.warn("Not a directory: " + directory);
        }
        //get supported parser extensions
        final Set<String> exts = new HashSet<String>(
            framework.getResourceFormatParserService().listSupportedFileExtensions());
        final File[] files = directory.listFiles(new FilenameFilter() {
            public boolean accept(final File file, final String s) {
                return exts.contains(
                    ResourceFormatParserService.getFileExtension(s));
            }
        });

        //set of previously cached file sources by file
        if (null != files) {
            //sort on filename
            Arrays.sort(files, null);
            synchronized (sourceCache) {
                final HashSet<File> trackedFiles = new HashSet<File>(sourceCache.keySet());
                for (final File file : files) {
                    //remove file that we want to keep
                    trackedFiles.remove(file);
                    if (!sourceCache.containsKey(file)) {
                        logger.debug("Adding new resources file to cache: " + file.getAbsolutePath());
                        try {
                            final ResourceModelSource source = createFileSource(project, file);
                            sourceCache.put(file, source);

                        } catch (ExecutionServiceException e) {
                            e.printStackTrace();
                            logger.debug("Failed adding file " + file.getAbsolutePath() + ": " + e.getMessage(), e);
                        }
                    }
                }
                //remaining trackedFiles are files that have been removed from the dir
                for (final File oldFile : trackedFiles) {
                    logger.debug("Removing from cache: " + oldFile.getAbsolutePath());
                    sourceCache.remove(oldFile);
                }
            }
        }
    }