We accept sloppy urls and attempt to clean them up
10274
def parse_url(url)
      s = url

      # remove entities
      s = HTML_ENTITIES.decode(s)

      # fix a couple of common encoding bugs
      s = s.gsub(' ', '%20')
      s = s.gsub("'", '%27')

      # append query manually (instead of letting HTTParty handle it) so we can
      # include it in cache_key
      query = options.delete(:query)
      if query.present?
        q = HTTParty::HashConversions.to_params(query)
        separator = s.include?('?') ? '&' : '?'
        s = "#{s}#{separator}#{q}"
      end

      URI.parse(s)
    end
226928
function _cleanseInput(dirty){
      var clean = ''
      clean = dirty.replace('@', '')
      clean = clean.replace('#', '')
      clean = clean.replace(/\s/g, '')
      return clean
    }
1324427
def clean_url(self):
        
        url = self.cleaned_data['url']

        if url:
            pattern = re.compile(r'https?://(www\.)?instagr(\.am|am\.com)/p/\S+')
            if not pattern.match(url):
                raise forms.ValidationError('Please provide a valid instagram link.')

        return url
498180
def clean_text(self, url, **kwargs):
        """"""
        try:
            return normalize_url(url)
        except UnicodeDecodeError:
            log.warning("Invalid URL: %r", url)
438150
function cleanup(html) {
      html = cheerio.load(html);
      html('[' + ignoreAttr + ']').removeAttr(ignoreAttr);
      html = html.html();
      return html;
    }
291215
def clean_url(self):
        """"""
        raw_url = self.request['url']
        parsed_url = urlparse(raw_url)
        qsl = parse_qsl(parsed_url.query)
        for qs in qsl:
            new_url = self._join_url(parsed_url,
                                     [i for i in qsl if i is not qs])
            new_request = deepcopy(self.request)
            new_request['url'] = new_url
            self._add_task('qsl', qs, new_request)
        return self
679241
private function cleanUrl($url)
    {
        if (strlen($url) > 1 && strrpos($url, '/') === strlen($url) - 1) {
            $url = substr($url, 0, -1);
        }
        if (strrpos($url, '.php') === strlen($url) - 4) {
            $url = substr($url, 0, -4);
        }
        return $url;
    }
858564
def clean_tenant_url(url_string):
    
    if hasattr(settings, 'PUBLIC_SCHEMA_URLCONF'):
        if (settings.PUBLIC_SCHEMA_URLCONF and
                url_string.startswith(settings.PUBLIC_SCHEMA_URLCONF)):
            url_string = url_string[len(settings.PUBLIC_SCHEMA_URLCONF):]
    return url_string
643257
function normalizeUrl(url, pathStripRe) {
  if (url.indexOf('/') !== -1) {
    return url.replace(/^file\:\/\//, '').replace(pathStripRe, '');
  } else {
    return '/' + url;
  }
}
168123
def links(self):
        
        def clean(url):
            "Tidy up an URL."
            scheme, netloc, path, params, query, frag = urlparse(url)
            return urlunparse((scheme, netloc, quote(path),
                               params, query, frag))

        result = set()
        for match in self._href.finditer(self.data):
            d = match.groupdict('')
            rel = (d['rel1'] or d['rel2'] or d['rel3'] or
                   d['rel4'] or d['rel5'] or d['rel6'])
            url = d['url1'] or d['url2'] or d['url3']
            url = urljoin(self.base_url, url)
            url = unescape(url)
            url = self._clean_re.sub(lambda m: '%%%2x' % ord(m.group(0)), url)
            result.add((url, rel))
        # We sort the result, hoping to bring the most recent versions
        # to the front
        result = sorted(result, key=lambda t: t[0], reverse=True)
        return result
483485
def prune_urls(url_set, start_url, allowed_list, ignored_list):
    """"""
    result = set()

    for url in url_set:
        allowed = False
        for allow_url in allowed_list:
            if url.startswith(allow_url):
                allowed = True
                break

        if not allowed:
            continue

        ignored = False
        for ignore_url in ignored_list:
            if url.startswith(ignore_url):
                ignored = True
                break

        if ignored:
            continue

        prefix, suffix = (url.rsplit('.', 1) + [''])[:2]
        if suffix.lower() in IGNORE_SUFFIXES:
            continue

        result.add(url)

    return result
1034938
function (url) {
            var tmp = url.replace(/\burl\s*/gi, "");    // removing url word
            tmp = tmp.charAt(0) === "(" ? tmp.substring(1, tmp.length - 1) : tmp; // removing brackets
            tmp = tmp.charAt(0) === "\'" || tmp.charAt(0) === "\"" ? tmp.substring(1, tmp.length - 1) : tmp; // removing quotes
            return tmp;
        }
341508
def sanitize_http_request_querystring(client, event):
    
    try:
        query_string = force_text(event["context"]["request"]["url"]["search"], errors="replace")
    except (KeyError, TypeError):
        return event
    if "=" in query_string:
        sanitized_query_string = _sanitize_string(query_string, "&", "=")
        full_url = event["context"]["request"]["url"]["full"]
        event["context"]["request"]["url"]["search"] = sanitized_query_string
        event["context"]["request"]["url"]["full"] = full_url.replace(query_string, sanitized_query_string)
    return event
391969
function correctHttpAndHttps(url) {
	var baseURL = url;

	// Check if http:// is correct
	if (new RegExp('^http://').test(baseURL)) {
		// Do nothing

	} else {
		// Check if https:// is correct
		if (new RegExp('^https://').test(baseURL)) {
			// Do nothing

		} else {
			// http:// or https:// is not set correct

			// Replace protocol mistakes like 'ttp://' to 'http://'
			var testHttp01 = new RegExp('^ttp://');
			if (testHttp01.test(baseURL)) {
				baseURL = baseURL.replace(testHttp01, 'http://')
			} else {
				var testHttp02 = new RegExp('^tp://');
				if (testHttp02.test(baseURL)) {
					baseURL = baseURL.replace(testHttp02, 'http://')
				} else {
					var testHttp03 = new RegExp('^p://');
					if (testHttp03.test(baseURL)) {
						baseURL = baseURL.replace(testHttp03, 'http://')
					} else {
						var testHttp04 = new RegExp('^://');
						if (testHttp04.test(baseURL)) {
							baseURL = baseURL.replace(testHttp04, 'http://')
						} else {
							var testHttp05 = new RegExp('^//');
							if (testHttp05.test(baseURL)) {
								baseURL = baseURL.replace(testHttp05, 'http://')
							} else {
								var testHttp06 = new RegExp('^/');
								if (testHttp06.test(baseURL)) {
									baseURL = baseURL.replace(testHttp06, 'http://')
								} else {
									var testHttps01 = new RegExp('^ttps://');
									if (testHttps01.test(baseURL)) {
										baseURL = baseURL.replace(testHttps01, 'https://')
									} else {
										var testHttps02 = new RegExp('^tps://');
										if (testHttps02.test(baseURL)) {
											baseURL = baseURL.replace(testHttps02, 'https://')
										} else {
											var testHttps03 = new RegExp('^ps://');
											if (testHttps03.test(baseURL)) {
												baseURL = baseURL.replace(testHttps03, 'https://')
											} else {
												var testHttps04 = new RegExp('^s://');
												if (testHttps04.test(baseURL)) {
													baseURL = baseURL.replace(testHttps04, 'https://')
												} else {
													var testHttp07 = new RegExp('^.');
													if (testHttp07.test(baseURL)) {
														baseURL = 'http://' + baseURL;
													} else {
														// Other mistakes
													}
												}
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}

	return baseURL;
}
262901
public function cleanUrlHandler(): void
    {
        global $pretext;

        $basename = explode('?', (string) $pretext['request_uri']);
        $basename = basename(array_shift($basename));

        if ($basename === 'robots.txt') {
            $this->sendRobots();
        }

        if ($basename === 'sitemap.xml' || $basename === 'sitemap.xml.gz') {
            $this->populateArticleFields()->sendSitemap();
        }
    }
1104702
function(urls){
   

		if(this.hCards.length > -1){

			// remove all organisational hCards using fn = org.organization-name pattern
        	var i = this.hCards.length;
			while (i--) {
				if(this.hCards[i].org && this.hCards[i].org[0]['organization-name']){
	            	if( this.hCards[i].fn === this.hCards[i].org[0]['organization-name'] ){
	            		this.hCards.splice(i,1);
	            	}
            	}
			}


			// if there is only one hcard use that
		    if(this.hCards.length === 1){
		        this.representativehCard = this.hCards[0];
		    }else{
				i = this.hCards.length;
				var x = 0;
				while (x < i) {
				    var hcard = this.hCards[x];
				    if(hcard.url){
				    	// match the urls of the hcard to the known urls
						for (var y = 0; y < urls.length; y++) {
							for (var z = 0; z < hcard.url.length; z++) {
						    	if( utils.compareUrl(urls[y], hcard.url[z]) ){
						    		this.representativehCard = hcard;
						    	}
							}
						}
				    	// match the xfn urls of the page to hcard urls
						if(this.xfn.length > -1){
							for (var z = 0; z < this.xfn.length; z++) {
								for (var y = 0; y < hcard.url.length; y++) {
							    	if( this.xfn[z].link !== undefined 
							    		&& this.xfn[z].rel !== undefined
							    		&& utils.compareUrl(hcard.url[y], this.xfn[z].link) 
							    		&& this.xfn[z].rel === 'me'){
							    		this.representativehCard = hcard;
							    	}
								}
							}
						}
				    }
				    x++;
				}
		    }
		}		


	    // if we have a hresume use contact hcard
		if(this.hResumes.length !== 0){
			if(this.hResumes[0].contact){
				this.representativehCard = this.hResumes[0].contact
				// description
				if(this.hResumes[0].summary !== '')  
					this.representativehCard.note = this.hResumes[0].summary;
				// job role

				// org


			}
		}
		this.filterRepresentativehCard();

	}
843628
def clean_url(url, base_url=None):
    """"""
    parsed_url = urlparse(url)

    fragment = '{url.fragment}'.format(url=parsed_url)
    if fragment:
        url = url.split(fragment)[0]

    # Identify internal URLs and fix their format
    netloc = '{url.netloc}'.format(url=parsed_url)
    if base_url is not None and not netloc:
        parsed_base = urlparse(base_url)
        split_base = '{url.scheme}://{url.netloc}{url.path}/'.format(url=parsed_base)
        url = urljoin(split_base, url)
        netloc = '{url.netloc}'.format(url=urlparse(url))

    if 'www.' in netloc:
        url = url.replace(netloc, netloc.replace('www.', ''))
    return url.rstrip(string.punctuation)