Return the 'pages' from the starting url
    Technically, look for the 'next 50' link, yield and download it,  repeat
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
364072
def links(cls, page):
    """"""
    for match in cls.HREF_RE.finditer(page):
      yield cls.href_match_to_url(match)
697793
def get_index_pages(original_url, pagination_index, pagination_interval, options={})
      resps = []
      prev_url = nil
      while resps.empty? || !resps.last.empty?
        next_url = self.next_index_page_url(original_url, pagination_index)
        break if next_url.empty?
        
        next_url = resolve_url(next_url, original_url)
        break if next_url == prev_url

        next_resp = self.get_page(next_url, @index_debug, options).to_s
        prev_url = next_url
        pagination_index += pagination_interval
        resps << next_resp
      end
      resps
    end
354081
def get(url, last_modified=None):
    
    try:
        with closing(urllib2.urlopen(url)) as page:
            if last_modified is not None:
                last_mod = dateutil.parser.parse(dict(page.info())['last-modified'])
                if last_mod <= last_modified:
                    return ""
            return page.read()
    except urllib2.URLError:
        return ""
1078831
function pullAllPages() {
        // 1. Pull one page
        // 2. Check if Pull is complete
        // 3. If it is complete, go to 5. If not, update the query to fetch the next page.
        // 4. Go to 1
        // 5. DONE
        return pullPage().then(function(pulledRecords) {
            if (!isPullComplete(pulledRecords)) {
                // update query and continue pulling the remaining pages
                return updateQueryForNextPage(pulledRecords).then(function() {
                    return pullAllPages();
                });
            }
        });
    }
21065
def download_urls(cls, urls, force_download=False):
        
        for url in urls:
            file_path = cls.get_path_to_file_from_url(url)

            if os.path.exists(file_path) and not force_download:
                log.info('already downloaded %s to %s', url, file_path)
            else:
                log.info('downloading %s to %s', url, file_path)
                download_timer = time.time()
                urlretrieve(url, file_path)
                log.info('downloaded in %.2f seconds', time.time() - download_timer)
1166223
def _get_pages(self, locations, project_name):
        # type: (Iterable[Link], str) -> Iterable[HTMLPage]
        
        seen = set()  # type: Set[Link]
        for location in locations:
            if location in seen:
                continue
            seen.add(location)

            page = _get_html_page(location, session=self.session)
            if page is None:
                continue

            yield page
249456
def _get_link_pages(page, per_page, count, page_url):
    # type: (int, int, int, str) -> Dict[str, str]
    
    current_page = _page_arg(page)
    links = {}
    end = page * per_page

    if page > 1:
        links['prev'] = page_url.replace(current_page, _page_arg(page - 1))

    if end < count:
        links['next'] = page_url.replace(current_page, _page_arg(page + 1))

    if per_page < count:
        links['first'] = page_url.replace(current_page, _page_arg(1))
        links['last'] = page_url.replace(current_page, _page_arg((count + per_page - 1) // per_page))

    return links
688311
def get_pages(self, include_draft=False):
        

        def pages_generator(pages_root_path):
            for file_path in traverse_directory(pages_root_path,
                                                yield_dir=False):
                rel_path = os.path.relpath(file_path, pages_root_path)
                rel_path, ext = os.path.splitext(rel_path)
                if not ext or ext == '.' or get_standard_format_name(
                        ext[1:]) is None:
                    continue  # pragma: no cover

                if rel_path.endswith(os.path.sep + 'index'):
                    rel_path = rel_path[:-len('index')]
                else:
                    rel_path += '.html'
                page = self.get_page(rel_path.replace(os.path.sep, '/'),
                                     include_draft=include_draft)
                if page is not None:
                    yield page

        pages_path = os.path.join(current_app.instance_path, 'pages')
        return list(pages_generator(pages_path))
3133
def _extract_next_page_link(self):
        

        # HEADS UP: we do not abort if next_page_link is already set:
        #           we try to find next (eg. find 3 if already at page 2).

        for pattern in self.config.next_page_link:
            items = self.parsed_tree.xpath(pattern)

            if not items:
                continue

            if len(items) == 1:
                item = items[0]

                if 'href' in item.keys():
                    self.next_page_link = item.get('href')

                else:
                    self.next_page_link = item.text.strip()

                LOGGER.info(u'Found next page link: %s.',
                            self.next_page_link)

                # First found link is the good one.
                break

            else:
                LOGGER.warning(u'%s items for next-page link %s',
                               items, pattern,
                               extra={'siteconfig': self.config.host})
1049553
def fetch_pages(url, referer = nil, depth = nil)
      begin
        url = URI(url) unless url.is_a?(URI)
        pages = []
        get(url, referer) do |response, code, location, redirect_to, response_time|
          pages << Page.new(location, :body => response.body.dup,
                                      :code => code,
                                      :headers => response.to_hash,
                                      :referer => referer,
                                      :depth => depth,
                                      :redirect_to => redirect_to,
                                      :response_time => response_time)
        end

        return pages
      rescue Exception => e
        if verbose?
          puts e.inspect
          puts e.backtrace
        end
        return [Page.new(url, :error => e)]
      end
    end
499435
def download_page(url, data=None):
    ''''''
    conn = urllib2.urlopen(url, data)
    resp = conn.read()
    conn.close()
    return resp
1140128
def first_page(self):
        
        try:
            start_url = self.start_url
        except AttributeError:
            raise PageError("Page %r is already the first page" % self)
        return self.page_for_url(start_url)
410014
def _fetch(self):
        
        while True:
            url = self._to_fetch.get()
            try:
                if url:
                    page = self.get_page(url)
                    if page is None:    # e.g. after an error
                        continue
                    for link, rel in page.links:
                        if link not in self._seen:
                            self._seen.add(link)
                            if (not self._process_download(link) and
                                self._should_queue(link, url, rel)):
                                logger.debug('Queueing %s from %s', link, url)
                                self._to_fetch.put(link)
            finally:
                # always do this, to avoid hangs :-)
                self._to_fetch.task_done()
            if not url:
                #logger.debug('Sentinel seen, quitting.')
                break
1177124
def get_book_links(links):
    
    book_links = []

    for link in links:
        data = DOWNER.download(link + "1")
        dom = dhtmlparser.parseString(data)

        book_links.extend(_parse_book_links(dom))

        max_page = _get_max_page(dom)
        if max_page == 1:
            continue

        for i in range(max_page - 1):
            data = DOWNER.download(link + str(i + 2))

            book_links.extend(
                _parse_book_links(
                    dhtmlparser.parseString(data)
                )
            )

    return book_links
380132
def next_url(request):
    
    next = request.GET.get("next", request.POST.get("next", ""))
    host = request.get_host()
    return next if next and is_safe_url(next, host=host) else None
157697
def get_next_page(self):
        """"""

        master_node = self._retrieve_next_page()

        seq = []
        for node in master_node.getElementsByTagName("track"):
            track = Track(
                _extract(node, "artist"),
                _extract(node, "name"),
                self.network,
                info={"image": _extract_all(node, "image")},
            )
            track.listener_count = _number(_extract(node, "listeners"))
            seq.append(track)

        return seq