Crawl URI using site config.
176669
def crawler(site, uri=None):
    """"""
    config = load_site_config(site)
    model = _get_model('crawl', config, uri)
    visited_set, visited_uri_set, consume_set, crawl_set = get_site_sets(
        site, config
    )

    if not visited_set.has(model.hash):
        visited_set.add(model.hash)
        visited_uri_set.add(model.uri)

        if (
            model.is_consume_page
            and not consume_set.has(model.hash)
        ):
            consume_set.add(model.hash)
            consume_q.enqueue(
                consumer,
                site,
                model.uri
            )
        else:
            for crawl_uri in model.uris_to_crawl:
                if (
                    not visited_uri_set.has(crawl_uri)
                    and not crawl_set.has(crawl_uri)
                ):
                    crawl_set.add(crawl_uri)
                    crawl_q.enqueue(
                        crawler,
                        site,
                        crawl_uri
                    )
777835
function urlConfig (url) {
        var config = {
            path: true,
            query: true,
            hash: true
        };

        if (!url) {
            return config;
        }

        if (RX_PROTOCOL.test(url)) {
            config.protocol = true;
            config.host = true;

            if (RX_PORT.test(url)) {
                config.port = true;
            }

            if (RX_CREDS.test(url)) {
                config.user = true;
                config.pass = true;
            }
        }

        return config;
    }
335521
def _run(self, url, auth):
        '''
        
        '''
        if url in self.visited:
            logger.debug("Skipping %s (already crawled)" % url)
            return
        self.visited.append(url)

        logger.info("Crawling: %s" % url)
        url = self._get_catalog_url(url)

        # Get an etree object
        xml_content = request_xml(url, auth)
        for ds in self._build_catalog(url, xml_content):
            yield ds
141323
def uri_from_fields(prefix, *fields):
        """"""
        string = '_'.join(AlignmentHelper.alpha_numeric(f.strip().lower(), '') for f in fields)

        if len(string) == len(fields) - 1:
            return ''

        return prefix + string
327948
def _parse_website_config(self, (response, xml_bytes)):
        """"""
        root = XML(xml_bytes)
        index_suffix = root.findtext("IndexDocument/Suffix")
        error_key = root.findtext("ErrorDocument/Key")

        return WebsiteConfiguration(index_suffix, error_key)
186464
def extract(self, url=None, raw_html=None):
        
        cc = CrawlCandidate(self.config, url, raw_html)
        return self.crawl(cc)
729826
def copy_uri_options(hosts, mongodb_uri):
        
        if "?" in mongodb_uri:
            options = mongodb_uri.split("?", 1)[1]
        else:
            options = None
        uri = "mongodb://" + hosts
        if options:
            uri += "/?" + options
        return uri
103998
def base_url(root):
    """"""
    for attr, value in root.attrib.iteritems():
        if attr.endswith('base') and 'http' in value:
            return value
    return None
273721
function site_url($uri = '')
	{
		if ($uri == '')
		{
			return $this->slash_item('base_url').$this->item('index_page');
		}

		if ($this->item('enable_query_strings') == FALSE)
		{
			$suffix = ($this->item('url_suffix') == FALSE) ? '' : $this->item('url_suffix');
			return $this->slash_item('base_url').$this->slash_item('index_page').$this->_uri_string($uri).$suffix;
		}
		else
		{
			return $this->slash_item('base_url').$this->item('index_page').'?'.$this->_uri_string($uri);
		}
	}
1193969
def discovery_uri(host = WORLD_WIDE_AUTHORITY)
      URI(DISCOVERY_TEMPLATE.expand(host: host, endpoint: authorize_endpoint))
    end
868836
def processURI(self, uri, prefix=''):
        
        components = urlparse.urlparse(uri)
        query = dict(urlparse.parse_qsl(components.query))
        bust = True
        bust &= bool(query)  # bust the cache if the query has stuff in it
        # bust the cache if the query key 'cache' isn't true
        bust &= query.get('cache') != 'true'
        return prefix + components.path, bust
1366906
def get_site_url(site_url=None, request=None):
    
    site_url = site_url or settings.get('SITE_URL')
    if site_url:
        return site_url
    elif request:
        return '{}://{}'.format(request.scheme, request.get_host())
    else:
        raise Exception('Either pass `site_url`, '
                        'or set `SITE_URL` in settings, '
                        'or pass `request` object.')
304140
protected function crawledBadUrls(): bool
    {
        return collect($this->urlsGroupedByStatusCode)->keys()->filter(function ($statusCode) {
            return ! $this->isSuccessOrRedirect($statusCode);
        })->count() > 0;
    }
580313
def sitemap_uri(self, basename):
        """"""
        if (re.match(r"\w+:", basename)):
            # looks like URI
            return(basename)
        elif (re.match(r"/", basename)):
            # looks like full path
            return(basename)
        else:
            # build from mapping with name appended
            return(self.mapper.default_src_uri() + '/' + basename)
1187634
def crawl(self, urls, name='crawl', api='analyze', **kwargs):
        
        # If multiple seed URLs are specified, join with whitespace.
        if isinstance(urls, list):
            urls = ' '.join(urls)
        url = self.endpoint('crawl')
        process_url = self.endpoint(api)
        params = {
            'token': self._token,
            'seeds': urls,
            'name': name,
            'apiUrl': process_url,
        }

        # Add any additional named parameters as accepted by Crawlbot
        params['maxToCrawl'] = 10
        params.update(kwargs)

        self._get(url, params=params)

        return Job(self._token, name, self._version)
25764
def adjust_uri(self, uri, relativeto):
        """"""

        key = (uri, relativeto)
        if key in self._uri_cache:
            return self._uri_cache[key]

        if uri[0] != '/':
            if relativeto is not None:
                v = self._uri_cache[key] = posixpath.join(
                                            posixpath.dirname(relativeto), uri)
            else:
                v = self._uri_cache[key] = '/' + uri
        else:
            v = self._uri_cache[key] = uri
        return v
577569
public static function site(string $site, string $uri = null): string
    {
        $otherSite = Application::getConfig('sites')->get($site);

        if ($otherSite === null) {
            throw new Exception("Unable to construct URL to site={$site}, site is not defined in configs/sites.php");
        }

        if ($uri === null || strlen($uri) === 0) {
            return $otherSite;
        } else {
            if ($uri[0] != '/') {
                $uri = '/' . $uri;
            }

            return $otherSite . $uri;
        }
    }