Try all permutations of hostname and path which can be applied

        to blacklisted URLs
510491
def url_permutations(url):
        
        def url_host_permutations(host):
            if re.match(r'\d+\.\d+\.\d+\.\d+', host):
                yield host
                return
            parts = host.split('.')
            l = min(len(parts), 5)
            if l > 4:
                yield host
            for i in range(l - 1):
                yield '.'.join(parts[i - l:])

        def url_path_permutations(path):
            yield path
            query = None
            if '?' in path:
                path, query = path.split('?', 1)
            if query is not None:
                yield path
            path_parts = path.split('/')[0:-1]
            curr_path = ''
            for i in range(min(4, len(path_parts))):
                curr_path = curr_path + path_parts[i] + '/'
                yield curr_path

        protocol, address_str = urllib.splittype(url)
        host, path = urllib.splithost(address_str)
        user, host = urllib.splituser(host)
        host, port = urllib.splitport(host)
        host = host.strip('/')
        seen_permutations = set()
        for h in url_host_permutations(host):
            for p in url_path_permutations(path):
                u = '{}{}'.format(h, p)
                if u not in seen_permutations:
                    yield u
                    seen_permutations.add(u)
1006390
def get_blacklist
      params = {
        :method => WebPurify::Constants.methods[:get_blacklist]
      }
      parsed = WebPurify::Request.query(text_request_base, @query_base, params)
      if parsed[:word]
        return [] << parsed[:word]
      else
        return []
      end
    end
1153946
private boolean acceptTargetUrl(String urlS) {
        // Whitelist has precedence. Only patterns on the white list are allowed
        if (whiteList != null) {
            return checkPattern(whiteList, urlS, true);
        }

        // Then blacklist: Everything on this list is forbidden
        if (blackList != null) {
            return checkPattern(blackList, urlS, false);
        }

        // If no list is configured, then everything is allowed
        return true;
    }
489951
def getHosts(filename=None, hostlist=None):
    """"""
    if filename:
        return getHostsFromFile(filename)
    elif hostlist:
        return getHostsFromList(hostlist)
    elif getEnv() == "SLURM":
        return getHostsFromSLURM()
    elif getEnv() == "PBS":
        return getHostsFromPBS()
    elif getEnv() == "SGE":
        return getHostsFromSGE()
    else:
        return getDefaultHosts()
533920
def do_blacklist(config, config_dir):
    
    if not os.path.exists(config_dir):
        print "Configuration '{}' does not exist.".format(config)
        exit(1)    

    editor = os.environ["EDITOR"]

    config_blacklist = os.path.join(config_dir, 'blacklist')
    call([editor, config_blacklist])
    print "Blacklist configuration has been updated."
459355
def splithost(url):
    """"""
    global _hostprog
    if _hostprog is None:
        import re
        _hostprog = re.compile('^//([^/?]*)(.*)$')

    match = _hostprog.match(url)
    if match:
        host_port = match.group(1)
        path = match.group(2)
        if path and not path.startswith('/'):
            path = '/' + path
        return host_port, path
    return None, url
969041
function getHostname(url, userOptions = {}) {
  const defaults = {
    removeSubdomains: true,
  };
  const options = Object.assign({}, defaults, userOptions);
  const domainRegExp = /^(?:[a-z]+:\/\/)?(?:[^/@]+@)?([^/:]+)/i;
  const ipAddressRegExp = /^\d{1,3}\.\d{1,3}.\d{1,3}\.\d{1,3}$/;
  const domainMatch = url.match(domainRegExp);

  if (domainMatch === null) {
    throw new Error(`URL is invalid: ${url}`);
  }

  // If the hostname is an IP address, no further processing can be done.
  const hostname = domainMatch[1];
  if (ipAddressRegExp.test(hostname)) {
    return hostname;
  }

  // Return the hostname with subdomains removed, if requested.
  return (options.removeSubdomains) ? removeSubdomains(hostname) : hostname;
}
942269
protected function get_blacklisted_hosts_by_category() {
        // For each of the admin setting entries, check and place in the correct section of the config array.
        $config = ['ipv6' => [], 'ipv4' => [], 'domain' => [], 'domainwildcard' => []];
        $entries = $this->get_blacklisted_hosts();
        foreach ($entries as $entry) {
            if (ip_utils::is_ipv6_address($entry) || ip_utils::is_ipv6_range($entry)) {
                $config['ipv6'][] = $entry;
            } else if (ip_utils::is_ipv4_address($entry) || ip_utils::is_ipv4_range($entry)) {
                $config['ipv4'][] = $entry;
            } else if (ip_utils::is_domain_name($entry)) {
                $config['domain'][] = $entry;
            } else if (ip_utils::is_domain_matching_pattern($entry)) {
                $config['domainwildcard'][] = $entry;
            }
        }
        return $config;
    }
173253
function extractUrlTemplates(resourceName) {
                    if (hasUrlTemplate(resourceName)) {
                        var indexOfSlash = resourceName.indexOf("/");
                        return [resourceName.substr(0, indexOfSlash), resourceName.substr(indexOfSlash, resourceName.length)];
                    }
                }
815028
function printHostnames() {
  console.log(chalk.yellow.bold(`${indent(2)} Hostnames requested: `));

  if (Object.keys(hostnames).length === 0) {
    console.log(chalk.yellow(`${indent(4)}none`));
    return;
  }

  for (const key in hostnames) {
    if (!hostnames[key]) return;
    console.log(chalk.yellow(`${indent(4)}${key}: ${hostnames[key]}`));
  }
}
161313
def is_blacklisted(self, appid, crawlid):
        '''
        
        '''
        key_check = '{appid}||{crawlid}'.format(appid=appid,
                                                crawlid=crawlid)
        redis_key = self.spider.name + ":blacklist"
        return self.redis_conn.sismember(redis_key, key_check)
551488
def chopurl(url):
    '''
    
    '''
    ret = {}
    if url.find('://') == -1:
        raise s_exc.BadUrl(':// not found in [{}]!'.format(url))

    scheme, remain = url.split('://', 1)
    ret['scheme'] = scheme.lower()

    # carve query params from the end
    if remain.find('?') != -1:
        query = {}
        remain, queryrem = remain.split('?', 1)

        for qkey in queryrem.split('&'):
            qval = None
            if qkey.find('=') != -1:
                qkey, qval = qkey.split('=', 1)

            query[qkey] = qval

        ret['query'] = query

    pathrem = ''
    slashoff = remain.find('/')
    if slashoff != -1:
        pathrem = remain[slashoff:]
        remain = remain[:slashoff]

    # detect user[:passwd]@netloc syntax
    if remain.find('@') != -1:
        user, remain = remain.rsplit('@', 1)
        if user.find(':') != -1:
            user, passwd = user.split(':', 1)
            ret['passwd'] = passwd

        ret['user'] = user

    # remain should be down to host[:port]

    # detect ipv6 [addr]:port syntax
    if remain.startswith('['):
        hostrem, portstr = remain.rsplit(':', 1)
        ret['port'] = int(portstr)
        ret['host'] = hostrem[1:-1]

    # detect ipv6 without port syntax
    elif remain.count(':') > 1:
        ret['host'] = remain

    # regular old host or host:port syntax
    else:

        if remain.find(':') != -1:
            remain, portstr = remain.split(':', 1)
            ret['port'] = int(portstr)

        ret['host'] = remain

    ret['path'] = pathrem
    return ret
743825
def _sub_hostname(self, line):
        '''
        
        '''
        try:
            for od,d in self.dn_db.items():
                #regex = re.compile(r'\w*\.%s' % d)
                regex = re.compile(r'(?![\W\-\:\ \.])[a-zA-Z0-9\-\_\.]*\.%s' % d)
                hostnames = [each for each in regex.findall(line)]
                if len(hostnames) > 0:
                    for hn in hostnames:
                        new_hn = self._hn2db(hn)
                        self.logger.debug("Obfuscating FQDN - %s > %s", hn, new_hn)
                        line = line.replace(hn, new_hn)
            if self.hostname:
                line = line.replace(self.hostname, self._hn2db(self.hostname))  #catch any non-fqdn instances of the system hostname

            return line
        except Exception as e: # pragma: no cover
            self.logger.exception(e)
            raise Exception('SubHostnameError: Unable to Substitute Hostname/Domainname')
242319
def attempt_open_query_permutations(url, orig_file_path, is_header_file):
    
    directory = dirname(convert_to_platform_safe(orig_file_path)) + "/"

    # get all filenames in directory
    try:
        filenames = [f for f in os.listdir(directory)
                     if isfile(join(directory, f))]
    except OSError:
        return

    # ensure that there are not extra parameters on any files
    if is_header_file:
        filenames = [f for f in filenames if ".http-headers" in f]
        filenames = [f for f in filenames if
                     _compare_file_name(orig_file_path + ".http-headers",
                                        directory,
                                        f)]
    else:
        filenames = [f for f in filenames if ".http-headers" not in f]
        filenames = [f for f in filenames if _compare_file_name(orig_file_path,
                                                                directory,
                                                                f)]

    url_parts = url.split("/")
    url_parts = url_parts[len(url_parts) - 1].split("?")

    base = url_parts[0]
    params = url_parts[1]

    params = params.split("&")

    # check to ensure that the base url matches
    filenames = [f for f in filenames if f.startswith(base)]

    params = [convert_to_platform_safe(unquote(p)) for p in params]

    # ensure that all parameters are there
    for param in params:
        filenames = [f for f in filenames if param in f]

    # if we only have one file, return it
    if len(filenames) == 1:
        path = join(directory, filenames[0])
        return open_file(path)

    # if there is more than one file, raise an exception
    if len(filenames) > 1:
        raise DataFailureException(url,
                                   "Multiple mock data files matched the " +
                                   "parameters provided!",
                                   404)
785442
def get_urlpatterns(self):
        
        return [
            path(
                _('mark/forums/'),
                self.mark_forums_read_view.as_view(),
                name='mark_all_forums_read',
            ),
            path(
                _('mark/forums/<int:pk>/'),
                self.mark_forums_read_view.as_view(),
                name='mark_subforums_read',
            ),
            path(
                _('mark/forum/<int:pk>/topics/'),
                self.mark_topics_read_view.as_view(),
                name='mark_topics_read',
            ),
            path(
                _('unread-topics/'),
                self.unread_topics_view.as_view(),
                name='unread_topics',
            ),
        ]
952316
def urlmap(patterns):
    """"""
    for pattern in patterns:
        group = getattr(pattern, 'namespace', None)
        if group is None:
            group = getattr(pattern, 'app_name', None)
        path = '/' + get_pattern(pattern).lstrip('^').rstrip('$')
        if isinstance(pattern, PATTERNS):
            yield (group, pattern.name), path
        elif isinstance(pattern, RESOLVERS):
            subpatterns = pattern.url_patterns
            for (_, name), subpath in urlmap(subpatterns):
                yield (group, name), path.rstrip('/') + subpath
1266686
function checkUrl(){
    var newPath = location.hash.substr(1) || '';

    if (newPath != currentPath)
    {
      // check for recursion
      if (preventRecursion(newPath))
        return;

      // save current path
      currentPath = newPath;

      var routesToLeave = [];
      var routesToEnter = [];
      var routesToMatch = [];

      allRoutes.forEach(function(route){
        var flags = route.processLocation_(newPath);

        if (flags & ROUTE_LEAVE)
          routesToLeave.push(route);
        if (flags & ROUTE_ENTER)
          routesToEnter.push(route);
        if (flags & ROUTE_MATCH)
          routesToMatch.push(route);
      });

      for (var i = 0; i < routesToLeave.length; i++)
        routeLeave(routesToLeave[i]);

      for (var i = 0; i < routesToEnter.length; i++)
        routeEnter(routesToEnter[i]);

      for (var i = 0; i < routesToMatch.length; i++)
        routeMatch(routesToMatch[i]);

      /** @cut */ flushLog(namespace + ': hash changed to "' + newPath + '"');
    }
    else
    {
      allRoutes.forEach(function(route){
        if (route.value)
        {
          routeEnter(route, true);
          routeMatch(route, true);
        }
      });

      /** @cut */ flushLog(namespace + ': checkUrl()');
    }
  }