If given a single URL (as a String or URI), returns an Array of Pages which link to that URL
 If given an Array of URLs, returns a Hash (URI => [Page, Page...]) of Pages linking to those URLs
359016
def pages_linking_to(urls)
      unless urls.is_a?(Array)
        urls = [urls]
        single = true
      end

      urls.map! do |url|
        unless url.is_a?(URI)
          URI(url) rescue nil
        else
          url
        end
      end
      urls.compact

      links = {}
      urls.each { |url| links[url] = [] }
      values.each do |page|
        urls.each { |url| links[url] << page if page.links.include?(url) }
      end

      if single and !links.empty?
        return links[urls.first]
      else
        return links
      end
    end
696776
def retrieve_page(rurl = nil)
      durl = rurl || url
      return nil unless durl
      durl = filter_url(durl) if self.respond_to?(:filter_url)
      self.class.retrieve_session ||= WebPageParser::HTTP::Session.new
      encode(self.class.retrieve_session.get(durl))
    end
1226124
def fetch_pages(url, referer = nil, depth = nil, user_data = nil)
      url = URI(url)
      pages = []
      get(url, referer) do |response, code, location, redirect_to, response_time|
        handle_compression response
        page = Page.new(location, body: response.body,
                                  code: code,
                                  headers: response.to_hash,
                                  referer: referer,
                                  depth: depth,
                                  redirect_to: redirect_to,
                                  response_time: response_time,
                                  fetched_at: Time.now.to_i)
        page.user_data = user_data unless user_data.nil?
        pages << page
      end
      pages
    rescue *RESCUABLE_ERRORS => e
      if verbose?
        puts e.inspect
        puts e.backtrace
      end
      page = Page.new(url, error: e, referer: referer, depth: depth)
      page.user_data = user_data unless user_data.nil?
      [page]
    end
898443
function linkInPage (link) {
    const pageUrl = location.hash
      ? stripHash(location.href)
      : location.href;

    return link.tagName.toLowerCase() === 'a'
      && link.hash.length > 0
      && stripHash(link.href) === pageUrl;
  }
33718
def parse_url(url, options)
      url.gsub!(/\s/, '')
      url.gsub!(/\?.*/, '') if options.strip_query?
      url.gsub!(/\#.*/, '') if options.strip_anchor?
      Addressable::URI.parse(url)
    end
1213527
function getUrlVars(string) {
  var vars = [],
    hash,
    hashes,
    str = string.toString();

  if (str.match('&#038;')) {
    str = str.replace("&#038;", "&");
  } else if (str.match('&#38;')) {
    str = str.replace("&#38;", "&");
  } else if (str.match('&amp;')) {
    str = str.replace("&amp;", "&");
  }
  if (str.match('#')) {
    str = str.split('#')[0];
  }
  hashes = str.slice(str.indexOf('?') + 1).split('&');

  for(var i = 0; i < hashes.length; i++) {
    hash = hashes[i].split('=');
    vars.push(hash[0]);
    vars[hash[0]] = hash[1];
  }

  return vars;
}
48670
def lookup_url(self, url):
        """"""
        if type(url) is not str:
            url = url.encode('utf8')
        if not url.strip():
            raise ValueError("Empty input string.")
        url_hashes = URL(url).hashes
        try:
            list_names = self._lookup_hashes(url_hashes)
            self.storage.commit()
        except Exception:
            self.storage.rollback()
            raise
        if list_names:
            return list_names
        return None
1144546
def links(self):
        """"""
        # <https://example.com/?page=2>; rel="next", <https://example.com/?page=34>; rel="last"'
        # becomes
        # {
        #     'last': {'rel': 'last', 'url': 'https://example.com/?page=34'},
        #     'next': {'rel': 'next', 'url': 'https://example.com/?page=2'},
        # },
        result = {}
        if 'Link' in self.headers:
            value = self.headers['Link']

            for part in re.split(r', *<', value):
                link = {}
                vs = part.split(';')

                # First section is always an url.
                link['url'] = vs.pop(0).strip('\'" <>')

                for v in vs:
                    if '=' in v:
                        key, v = v.split('=')
                        link[key.strip('\'" ')] = v.strip('\'" ')

                rkey = link.get('rel') or link['url']
                result[rkey] = link

        return result
301116
def url_join(url, path):
    
    p = six.moves.urllib.parse.urlparse(url)

    t = None
    if p.path and p.path[-1] == '/':
        if path and path[0] == '/':
            path = path[1:]
        t = ''.join([p.path, path])
    else:
        t = ('' if path and path[0] == '/' else '/').join([p.path, path])

    return six.moves.urllib.parse.urlunparse(
        p[:2]+
        (t,)+ # os.sep is different on windows, don't use it here.
        p[3:]
    )
308041
def get_api_results(url)
      uri = URI.parse(url)
      puts "URL: #{url}" if self.verbose
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = true

      request = Net::HTTP::Get.new(uri.request_uri)
      response = http.request(request)

      JSON.parse(response.body)
    end
1126541
def links
      return @links unless @links.nil?
      @links = []
      return @links if !doc

      doc.search("//a[@href]").each do |a|
        u = a['href']
        next if u.nil? or u.empty?
        abs = to_absolute(u) rescue next
        @links << abs if in_domain?(abs)
      end
      @links.uniq!
      @links
    end
1089910
def links
      @links ||= begin
        links = HashWithIndifferentAccess.new
        self.link.each do |link|
          links[link['rel']] = link['href']
        end
        links
      end
    end
810177
def generate_page_link page_number
      url = core__add_param_to_url(@args[:url], @args[:param], page_number)
      if @args[:extra_params]
        @args[:extra_params].each do |key, value|
          url = core__add_param_to_url(url, key, value)
        end
      end
      url
    end
811794
function(options) {
        var pages = [];
        var fromPage = options.page - 2;
        if (fromPage < 2) {
          fromPage = 2;
        }
        for (var page = fromPage; (page < (fromPage + options.shown)); page++) {
          pages.push(page);
        }
        return pages;
      }
290920
def remote_a
      return unless @page_a_tags
      remote_a = []
      @page_a_tags.uniq.each do |link|
        uri = URI(link[0].to_ascii)
        if uri && @site_domain
          remote_a << link[0] unless uri.host == @site_domain
        end
      end
      remote_a
    end
69933
def paging
      page = 0
      results = []
      loop do
        page += 1
        current = yield(page)

        results += current

        break if (current || []).count < page_size # no more results
      end

      return results
    end
786717
def list_url?(u = nil, parsed: nil)
      return false if (u.nil? || u.empty?) && parsed.nil?
      parsed ||= parse_url(u)
      child_type = parsed[:child_type]
      parsed[:type] == 'lists' && (child_type.nil? || child_type.empty?)
    end