Set the URL referring to a robots.txt file.
762742
def set_url (self, url):
        """"""
        self.url = url
        self.host, self.path = urlparse.urlparse(url)[1:3]
605945
def _set_url(self, url):
    
    original_url = self._url
    try:
      self._update_index(url)
    except:
      self._url = original_url
      raise
604556
def load_robots_txt(self, url_info: URLInfo, text: str):
        ''''''
        key = self.url_info_key(url_info)
        parser = robotexclusionrulesparser.RobotExclusionRulesParser()
        parser.parse(text)

        self._parsers[key] = parser
1194026
private URL adaptURL(URL url) throws Exception {
        if (url != null && url.getProtocol().equals("file")) {
            url = new URL(fixEncodedURI(url.toURI().toASCIIString()));
        }
        return url;
    }
352255
def add_extra_urls(self, item_session: ItemSession):
        ''''''

        if item_session.url_record.level == 0 and self._sitemaps:
            extra_url_infos = (
                self.parse_url(
                    '{0}://{1}/robots.txt'.format(
                        item_session.url_record.url_info.scheme,
                        item_session.url_record.url_info.hostname_with_port)
                ),
                self.parse_url(
                    '{0}://{1}/sitemap.xml'.format(
                        item_session.url_record.url_info.scheme,
                        item_session.url_record.url_info.hostname_with_port)
                )
            )

            for url_info in extra_url_infos:
                item_session.add_child_url(url_info.url)
351785
public function setUrl()
	{
		$name = preg_replace("/[^a-zA-Z0-9]+/", " ", $this->getUsername());
		$name = preg_replace('!\s+!', ' ', trim($name));
		$this->url = '/authors/' . str_replace(" ", "-", strtolower($name)) . '.html';
	}
406102
def _set_repo(self, url):
        """"""
        if url.startswith('http'):
            try:
                self.repo = Proxy(url)
            except ProxyError, e:
                log.exception('Error setting repo: %s' % url)
                raise GritError(e)
        else:
            try:
                self.repo = Local(url)
            except NotGitRepository:
                raise GritError('Invalid url: %s' % url)
            except Exception, e:
                log.exception('Error setting repo: %s' % url)
                raise GritError(e)
1348389
def fetch_robots_txt(self, request: Request, file=None):
        '''
        '''
        url_info = request.url_info
        url = URLInfo.parse('{0}://{1}/robots.txt'.format(
            url_info.scheme, url_info.hostname_with_port)).url

        if not file:
            file = wpull.body.new_temp_file(os.getcwd(), hint='robots')

        with contextlib.closing(file):
            request = Request(url)

            session = self._web_client.session(request)
            while not session.done():
                wpull.util.truncate_file(file.name)

                try:
                    response = yield from session.start()
                    yield from session.download(file=file)
                except ProtocolError:
                    self._accept_as_blank(url_info)

                    return

            status_code = response.status_code

            if 500 <= status_code <= 599:
                raise ServerError('Server returned error for robots.txt.')

            if status_code == 200:
                self._read_content(response, url_info)
            else:
                self._accept_as_blank(url_info)
903604
def _urlretrieve(url, filename):
    
    ctx = create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = CERT_NONE

    with urlopen(url, context=ctx) as u, filename.open('wb') as f:
            f.write(u.read())
747690
def _allows_url (self, url_data, roboturl):
        """"""
        with cache_lock:
            if roboturl in self.cache:
                self.hits += 1
                rp = self.cache[roboturl]
                return rp.can_fetch(self.useragent, url_data.url)
            self.misses += 1
        kwargs = dict(auth=url_data.auth, session=url_data.session)
        if hasattr(url_data, "proxy") and hasattr(url_data, "proxy_type"):
            kwargs["proxies"] = {url_data.proxytype: url_data.proxy}
        rp = robotparser2.RobotFileParser(**kwargs)
        rp.set_url(roboturl)
        rp.read()
        with cache_lock:
            self.cache[roboturl] = rp
        self.add_sitemap_urls(rp, url_data, roboturl)
        return rp.can_fetch(self.useragent, url_data.url)
989109
def as_href(from_path)
      if @options.all_one_file
        "#" + path
      else
        RDoc::Markup::ToHtml.gen_relative_url from_path, path
      end
    end
911919
def substitute_ref_with_url(self, txt):
        

        # Find sphinx cross-references
        mi = re.finditer(r':([^:]+):`([^`]+)`', txt)
        if mi:
            # Iterate over match objects in iterator returned by re.finditer
            for mo in mi:
                # Initialize link label and url for substitution
                lbl = None
                url = None
                # Get components of current match: full matching text, the
                # role label in the reference, and the name of the
                # referenced type
                mtxt = mo.group(0)
                role = mo.group(1)
                name = mo.group(2)

                # If role is 'ref', the name component is in the form
                # label <name>
                if role == 'ref':
                    ma = re.match(r'\s*([^\s<]+)\s*<([^>]+)+>', name)
                    if ma:
                        name = ma.group(2)
                        lbl = ma.group(1)

                # Try to look up the current cross-reference. Issue a
                # warning if the lookup fails, and do the substitution
                # if it succeeds.
                try:
                    url = self.get_docs_url(role, name)
                    if role != 'ref':
                        lbl = self.get_docs_label(role, name)
                except KeyError as ex:
                    if len(ex.args) == 1 or ex.args[1] != 'role':
                        print('Warning: %s' % ex.args[0])
                else:
                    # If the cross-reference lookup was successful, replace
                    # it with an appropriate link to the online docs
                    rtxt = '[%s](%s)' % (lbl, url)
                    txt = re.sub(mtxt, rtxt, txt, flags=re.M)

        return txt
673417
public function set_url() {

		$plugins_url = plugins_url();
		$this_url    = trim( substr( trailingslashit( plugin_dir_url( __FILE__ ) ), strlen( $plugins_url ) ), '/' );

		if ( false !== strpos( $this_url, '/' ) ) {
			$url_path = explode( '/', $this_url );
			// generic 3 path depth: classes/namespace/ui|data.
			array_splice( $url_path, count( $url_path ) - 3 );
			$this_url = implode( '/', $url_path );
		}
		// setup the base URL.
		$this->url = trailingslashit( $plugins_url . '/' . $this_url );
	}
58718
def setUrl(self, url):
        
        if isUrl(url):
            self._url = url
        else:
            raise exceptions.BadUrlException(url)
        return self
256503
def add_sitemap_urls(self, rp, url_data, roboturl):
        """"""
        if not rp.sitemap_urls or not url_data.allows_simple_recursion():
            return
        for sitemap_url, line in rp.sitemap_urls:
            url_data.add_url(sitemap_url, line=line)
443872
def url_or_file_open(uri):
    """"""
    if (not re.match(r'''\w+:''', uri)):
        uri = 'file:' + uri
    return(urlopen(uri))
634006
def set_base_url(url):
    
    global _base_url, _dirty

    LOGGER.debug('Setting base URL to %s', url)
    _base_url = url
    _dirty = True