Gathers a list of URLs to ignore
516386
def url_ignore(self, project):
        
        project_list = False
        try:
            url_ignore = il['url_ignore']
        except KeyError:
            logger.error('Key Error processing url_ignore list values')

        try:
            project_exceptions = il.get('project_exceptions')
            for item in project_exceptions:
                if project in item:
                    exception_file = item.get(project)
                    with open(exception_file, 'r') as f:
                        url_list = yaml.safe_load(f)
                        project_list = url_list['url_ignore']
        except KeyError:
            logger.info('No url_ignore for %s', project)

        if project_list:
            url_ignore = url_ignore + project_list
            url_ignore_re = re.compile("|".join(url_ignore), flags=re.IGNORECASE)
            return url_ignore_re
        else:
            url_ignore_re = re.compile("|".join(url_ignore), flags=re.IGNORECASE)
            return url_ignore_re
1018699
def _depaginate_all(self, url):
        
        items = []
        for x in self._depagination_generator(url):
            items += x
        return items
268299
def fetchUrls(cls, url, data, urlSearch):
        """"""
        searchUrls = []
        searches = makeSequence(urlSearch)
        for search in searches:
            for match in search.finditer(data[0]):
                searchUrl = match.group(1)
                if not searchUrl:
                    raise ValueError("Pattern %s matched empty URL at %s." % (search.pattern, url))
                out.debug(u'matched URL %r with pattern %s' % (searchUrl, search.pattern))
                searchUrls.append(normaliseURL(urljoin(data[1], searchUrl)))
            if searchUrls:
                # do not search other links if one pattern matched
                break
        if not searchUrls:
            patterns = [x.pattern for x in searches]
            raise ValueError("Patterns %s not found at URL %s." % (patterns, url))
        return searchUrls
168123
def links(self):
        
        def clean(url):
            "Tidy up an URL."
            scheme, netloc, path, params, query, frag = urlparse(url)
            return urlunparse((scheme, netloc, quote(path),
                               params, query, frag))

        result = set()
        for match in self._href.finditer(self.data):
            d = match.groupdict('')
            rel = (d['rel1'] or d['rel2'] or d['rel3'] or
                   d['rel4'] or d['rel5'] or d['rel6'])
            url = d['url1'] or d['url2'] or d['url3']
            url = urljoin(self.base_url, url)
            url = unescape(url)
            url = self._clean_re.sub(lambda m: '%%%2x' % ord(m.group(0)), url)
            result.add((url, rel))
        # We sort the result, hoping to bring the most recent versions
        # to the front
        result = sorted(result, key=lambda t: t[0], reverse=True)
        return result
1231009
def clean_link(self, url):
        """"""
        return self._clean_re.sub(lambda match: "%%%2x" % ord(match.group(0)), url)
277787
def clean_service_url(url):
    
    parts = urlparse(url)
    return urlunparse((parts.scheme, parts.netloc, parts.path, '', '', ''))
171578
def sanitize_url(url):
    """"""
    parts = urlparse(url)
    if parts.password is None:
        return url
    host_info = parts.netloc.rsplit('@', 1)[-1]
    parts = parts._replace(netloc='{}:{}@{}'.format(
        parts.username, REDACTED, host_info))
    return parts.geturl()
69945
def normalize_urls(urls):
    """"""
    _urls = []
    if isinstance(urls, list):
        if urls:
            if isinstance(urls[0], list):
                # multiple connections (list of the lists)
                _urls = urls
            elif isinstance(urls[0], str):
                # single connections (make it list of the lists)
                _urls = [urls]
        else:
            raise RuntimeError("No target host url provided.")
    elif isinstance(urls, str):
        _urls = [[urls]]
    return _urls
824859
def get_new_links(self, url, resp):
        """"""
        links_on_page = resp.xpath('//a/@href')
        links = [utils.clean_url(u, url) for u in links_on_page]

        # Remove non-links through filtering by protocol
        links = [x for x in links if utils.check_protocol(x)]

        # Restrict new URLs by the domain of the input URL
        if not self.args['nonstrict']:
            domain = utils.get_domain(url)
            links = [x for x in links if utils.get_domain(x) == domain]

        # Filter URLs by regex keywords, if any
        if self.args['crawl']:
            links = utils.re_filter(links, self.args['crawl'])
        return links
1009558
def get_ignored_tracks(self, localdir):
        
        ignore_file = "%s/.ignore" % localdir
        list = []
        if os.path.exists(ignore_file):
            f = open(ignore_file)
            ignored = f.readlines()
            f.close()

            for i in ignored:
                list.append("%s/%s" % (localdir, i.rstrip()))

        return list
1014892
def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """"""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result
632258
function getCleanUrl (url) {
  try {
    const parsed = new URL(url)
    return parsed.pathname || url
  } catch (err) {
    return url
  }
}
1035189
def ignore_path(path, ignore_list=None, whitelist=None):
    
    if ignore_list is None:
        return True

    should_ignore = matches_glob_list(path, ignore_list)
    if whitelist is None:
        return should_ignore

    return should_ignore and not matches_glob_list(path, whitelist)
1016430
def indexOf(self, url):
        
        for i, (m_url, _) in enumerate(self._stack):
            if m_url == url:
                return i
        return -1
329856
def get_urlpatterns(self):
        
        return [
            path('', self.index_view.as_view(), name='index'),
            path(
                _('forum/<str:slug>-<int:pk>/'),
                self.forum_view.as_view(),
                name='forum',
            ),
        ]
79335
private int getURLIndex(String url, List<String> urlList) {
        int urlIndex = 0;
        int numURLs = urlList.size();

        // get URL index
        if (url != null)
            for (int i = 0; i < numURLs; i++)
                if ((urlList.get(i)).equalsIgnoreCase(url)) {
                    urlIndex = i;
                    break;
                }

        return urlIndex;
    }
806890
def _clean_url(self, url):
        
        # TODO: Turn this into regex
        if not url.startswith('http') or url.endswith('}}') or 'nojs_router' in url:
            return None
        if site(norm(url).lower()) in config.NONCANONIC_SITES:
            clean_url = canonicalize_url(url, keep_params=True)
        else:
            clean_url = canonicalize_url(url)
        return clean_url