Return the URLs of all the links on a page together with information
        about their "rel" attribute, for determining which ones to treat as
        downloads and which ones to queue for further scraping.
168123
def links(self):
        
        def clean(url):
            "Tidy up an URL."
            scheme, netloc, path, params, query, frag = urlparse(url)
            return urlunparse((scheme, netloc, quote(path),
                               params, query, frag))

        result = set()
        for match in self._href.finditer(self.data):
            d = match.groupdict('')
            rel = (d['rel1'] or d['rel2'] or d['rel3'] or
                   d['rel4'] or d['rel5'] or d['rel6'])
            url = d['url1'] or d['url2'] or d['url3']
            url = urljoin(self.base_url, url)
            url = unescape(url)
            url = self._clean_re.sub(lambda m: '%%%2x' % ord(m.group(0)), url)
            result.add((url, rel))
        # We sort the result, hoping to bring the most recent versions
        # to the front
        result = sorted(result, key=lambda t: t[0], reverse=True)
        return result
1161311
def links(self):
    '''
    
    '''

    if not getattr(self, '_links', False):
      self._links = [
        link['title']
        for link in self.__continued_query({
          'prop': 'links',
          'plnamespace': 0,
          'pllimit': 'max'
        })
      ]

    return self._links
985390
def extract_original_links(base_url, bs4):
    
    valid_url = convert_invalid_url(base_url)

    url = urlparse(valid_url)

    base_url = '{}://{}'.format(url.scheme, url.netloc)

    base_url_with_www = '{}://www.{}'.format(url.scheme, url.netloc)

    links = extract_links(bs4)

    result_links = [anchor for anchor in links if anchor.startswith(base_url)]

    result_links_www = [anchor for anchor in links if anchor.startswith(base_url_with_www)]

    return list(set(result_links + result_links_www))
1152968
def get_google_links(limit, params, headers):
	
	links = []
	for start_index in range(0, limit, 10):
		params['start'] = start_index
		resp = s.get("https://www.google.com/search", params = params, headers = headers)
		page_links = scrape_links(resp.content, engine = 'g')
		links.extend(page_links)
	return links[:limit]
816677
def _get_links(self, response):
        
        html_text = response.text.encode('utf-8')
        doc = document_fromstring(html_text)
        links = []
        for e in doc.cssselect('a'):
            links.append(e.get('href'))
461079
async def get_all(self, url, params=None):
        
        if not params:
            params = {}
        items = []
        next_page_token = None

        while True:
            if next_page_token:
                params['pageToken'] = next_page_token
            response = await self.get_json(url, params=params)

            items.append(response)
            next_page_token = response.get('nextPageToken')
            if not next_page_token:
                break
        return items
539775
def a_urls(html):
    '''
    
    '''
    soup = BeautifulSoup(html, 'lxml')
    for node in soup.find_all('a'):
        try:
            href = node['href']
        except KeyError:
            continue
        yield norm_url(href)
1217229
def links(self):
        
        if self._links is None:
            self._links = list()
            self.__pull_combined_properties()
        return self._links
1250997
def get_page(url, stash=false, options={})
      return EMPTY_STRING if url.nil? || url.empty? #url is nil if the <a> lacks an `href` attribute.
      global_options = {
        :cache => stash,
        :verbose => @verbose
      }
      if @readable_filenames
        global_options[:readable_filenames] = true
      end
      if @stash_folder
        global_options[:readable_filenames] = true
        global_options[:cache_location] = @stash_folder
      end
      resp_and_cache = Downloader.new(url, global_options.merge(options)).get
      if resp_and_cache[:from_resource]
        puts "sleeping #{@sleep_time_between_requests} secs" if @verbose
        sleep @sleep_time_between_requests
      end
      resp_and_cache[:resp]
    end
64351
def links(self):
        
        if not self._responses:
            return None
        if 'Link' in self._responses[-1].headers:
            links = []
            for l in headers.parse_link(self._responses[-1].headers['Link']):
                link = {'target': l.target}
                link.update({k: v for (k, v) in l.parameters})
                links.append(link)
            return links
402679
def available_files(url):
        
        req = requests.get(url)

        if req.status_code != 200:
            raise base.FailedDownloadException('Failed to download data (status {}) from {}!'.format(req.status_code,
                                                                                                     url))

        page_content = req.text
        link_pattern = re.compile(r'<a href="(.*?)">(.*?)</a>')
        available_files = []

        for match in link_pattern.findall(page_content):
            if match[0].endswith('.tgz'):
                available_files.append(os.path.join(url, match[0]))

        return available_files
645985
def get_urls(self):
        
        parts = []
        seen = set()

        # Process item views in order
        for v in list(self._meta.item_views)+list(self._meta.action_views):
            if not v in seen:
                view, name = self.get_view_and_name(v)
                if view and name:
                    parts.append(self.get_url(name, view, v))
                seen.add(v)

        # Process everything else that we have not seen
        for v in set(self._views).difference(seen):
            # Get the url name
            view, name = self.get_view_and_name(v)
            if view and name:
                parts.append(self.get_url(name, view, v))

        return parts
1023336
def _extract_links(self, selector, response_url, response_encoding, base_url):
        '''
        
        '''
        links = []
        # hacky way to get the underlying lxml parsed document
        for el, attr, attr_val in self._iter_links(selector.root):
            # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
            try:
                attr_val = urljoin(base_url, attr_val)
            except ValueError:
                continue # skipping bogus links
            else:
                url = self.process_attr(attr_val)
                if url is None:
                    continue
            # added 'ignore' to encoding errors
            url = to_native_str(url, encoding=response_encoding,
                                errors='ignore')
            # to fix relative links after process_value
            url = urljoin(response_url, url)
            link = Link(url, _collect_string_content(el) or u'',
                        nofollow=rel_has_nofollow(el.get('rel')))
            links.append(link)
        return self._deduplicate_if_needed(links)
824859
def get_new_links(self, url, resp):
        """"""
        links_on_page = resp.xpath('//a/@href')
        links = [utils.clean_url(u, url) for u in links_on_page]

        # Remove non-links through filtering by protocol
        links = [x for x in links if utils.check_protocol(x)]

        # Restrict new URLs by the domain of the input URL
        if not self.args['nonstrict']:
            domain = utils.get_domain(url)
            links = [x for x in links if utils.get_domain(x) == domain]

        # Filter URLs by regex keywords, if any
        if self.args['crawl']:
            links = utils.re_filter(links, self.args['crawl'])
        return links
330138
def _get_links(self,):
        
        res = ''
        if self.links:
            for l in self.links:
                res += ' links = ' + str(l[0]) + '\n'
                if l[0].child_nodes:
                    for chld in l[0].child_nodes:
                        res += '   child = ' + str(chld) + '\n'
                if l[0].links:
                    for lnk in l[0].links:
                        res += '   sublink = ' + str(lnk[0]) + '\n'
        else:
            res += ' links = None\n'
            
        return res
959487
def scrape_links(self, text, context=False):
        ''''''
        return self.iter_processed_links(io.StringIO(text), context=context)
364597
def all_base_links(rel_to=None):
    """"""
    links = [
        base_link('get-releases'),
        base_link('service'),
        server_link('get-servers'),
        server_link('add-server'),
        replica_set_link('add-replica-set'),
        replica_set_link('get-replica-sets'),
        sharded_cluster_link('add-sharded-cluster'),
        sharded_cluster_link('get-sharded-clusters')
    ]
    for link in links:
        if link['rel'] == rel_to:
            link['rel'] = 'self'
    return links