Parse all search result pages.
512286
def get_results():
    """"""
    # store info in a dictionary {name -> shortname}
    res = {}
    session = requests.Session()
    handle_url('http://www.gocomics.com/features', session, res)
    handle_url('http://www.gocomics.com/explore/espanol', session, res)
    handle_url('http://www.gocomics.com/explore/editorial_list', session, res)
    handle_url('http://www.gocomics.com/explore/sherpa_list', session, res)
    save_result(res, json_file)
694619
def fetch_result(self):
        """"""
        results = self.soup.find_all('div', {'class': 'container container-small'})
        href = None
        is_match = False

        i = 0
        while i < len(results) and not is_match:
            result = results[i]
            anchor = result.find('a', {'rel': 'bookmark'})
            is_match = self._filter_results(result, anchor)
            href = anchor['href']

            i += 1

        try:
            page = get_soup(href)
        except (Exception):
            page = None

        # Return page if search is successful
        if href and page:
            return page
        else:
            raise PageNotFoundError(PAGE_ERROR)
1261929
def all(page = 1, limit = 100, latest = true)
      opts = connection.version == 2 ? {:unseen => latest} : {:latest => latest}
      connection.get(resource_uri, {:query => {:limit => limit, :page => page}.merge(opts)})
    end
1078831
function pullAllPages() {
        // 1. Pull one page
        // 2. Check if Pull is complete
        // 3. If it is complete, go to 5. If not, update the query to fetch the next page.
        // 4. Go to 1
        // 5. DONE
        return pullPage().then(function(pulledRecords) {
            if (!isPullComplete(pulledRecords)) {
                // update query and continue pulling the remaining pages
                return updateQueryForNextPage(pulledRecords).then(function() {
                    return pullAllPages();
                });
            }
        });
    }
818135
@Override
	public void search(
		String[] words,
		WebSiteRequest req,
		HttpServletResponse response,
		List<SearchResult> results,
		AoByteArrayOutputStream bytes,
		List<WebPage> finishedPages
	) {
	}
766042
def all_titles
      result = []
      @pages.each do |page|
        result << [page.page_url, page.all_titles] if page.page_a_tags
      end
      result
    end
279124
def get_all(self, search_filter=None):
        """"""
        items = self.backend.get_all()

        if not items:
            if self.version == 1:
                return {self.namespace: []}
            return []

        if search_filter:
            items = jmespath.search(search_filter, items)

        return items
1244012
def parse_list(self, entries):
        """"""
        result_entries = SearchableList()
        for entry in entries:
            result_entries.append(self.instance.parse(self.requester, entry))
        return result_entries
268299
def fetchUrls(cls, url, data, urlSearch):
        """"""
        searchUrls = []
        searches = makeSequence(urlSearch)
        for search in searches:
            for match in search.finditer(data[0]):
                searchUrl = match.group(1)
                if not searchUrl:
                    raise ValueError("Pattern %s matched empty URL at %s." % (search.pattern, url))
                out.debug(u'matched URL %r with pattern %s' % (searchUrl, search.pattern))
                searchUrls.append(normaliseURL(urljoin(data[1], searchUrl)))
            if searchUrls:
                # do not search other links if one pattern matched
                break
        if not searchUrls:
            patterns = [x.pattern for x in searches]
            raise ValueError("Patterns %s not found at URL %s." % (patterns, url))
        return searchUrls
1272056
def search(self, query, fields=None, page=1, max_records=None, flatten=True):
        """"""
        if fields is None:
            fields = []
        page = int(page)
        pages = float('inf')
        data = {
            "query": query,
            "page": page,
            "fields": fields,
            "flatten": flatten
        }

        count = 0
        while page <= pages:
            payload = self._post(self.search_path, data=data)
            pages = payload['metadata']['pages']
            page += 1
            data["page"] = page

            for result in payload["results"]:
                yield result
                count += 1
                if max_records and count >= max_records:
                    return
1137271
def Parse(self):
    """"""
    for timestamp, url, title in self.Query(self.VISITS_QUERY):
      if not isinstance(timestamp, (long, int)):
        timestamp = 0

      yield [timestamp, "FIREFOX3_VISIT", url, title]
319773
def _parse_page(self, query):
        
        args = {k[5:-1]: v for k, v in query.items() if k.startswith('page[')}

        if {'number', 'size'} == set(args.keys()):
            if not args['number'].isdecimal() or not args['size'].isdecimal():
                raise BadRequestError('Page query parameters must be integers')

            number = int(args['number'])
            size = int(args['size'])
            start = number * size

            return start, start + size - 1

        if {'limit', 'offset'} == set(args.keys()):
            if not args['limit'].isdecimal() or not args['offset'].isdecimal():
                raise BadRequestError('Page query parameters must be integers')

            limit = int(args['limit'])
            offset = int(args['offset'])

            return offset, offset + limit - 1

        return 0, None
734072
def all(self):
        
        page = self._load(self.href)
        while True:
            try:
                for item in page._items:
                    yield item
                page = page.next_page()
            except PaginationError:
                break
835508
async def parseResults(self, api_data):
    
    results = []

    # parse HTML and get results
    parser = lxml.etree.HTMLParser()
    html = lxml.etree.XML(api_data.decode("latin-1"), parser)

    for rank, result in enumerate(__class__.RESULTS_SELECTOR(html), 1):
      # extract url
      metadata_div = result.find("div")
      metadata_json = lxml.etree.tostring(metadata_div, encoding="unicode", method="text")
      metadata_json = json.loads(metadata_json)
      google_url = result.find("a").get("href")
      if google_url is not None:
        query = urllib.parse.urlsplit(google_url).query
      else:
        query = None
      if not query:
        img_url = metadata_json["ou"]
      else:
        query = urllib.parse.parse_qs(query)
        img_url = query["imgurl"][0]
      # extract format
      check_metadata = CoverImageMetadata.NONE
      format = metadata_json["ity"].lower()
      try:
        format = SUPPORTED_IMG_FORMATS[format]
      except KeyError:
        # format could not be identified or is unknown
        format = None
        check_metadata = CoverImageMetadata.FORMAT
      # extract size
      if not query:
        size = metadata_json["ow"], metadata_json["oh"]
      else:
        size = tuple(map(int, (query["w"][0], query["h"][0])))
      # extract thumbnail url
      thumbnail_url = metadata_json["tu"]
      # result
      results.append(GoogleImagesCoverSourceResult(img_url,
                                                   size,
                                                   format,
                                                   thumbnail_url=thumbnail_url,
                                                   source=self,
                                                   rank=rank,
                                                   check_metadata=check_metadata))

    return results
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
219171
function parseSearchResults(err, body) {
    if (err) { return tubesio.finish(err); }

    var result = [];

    jsdom.env({
        html: body,
        scripts: [
            'http://code.jquery.com/jquery-1.5.min.js'
        ]
    }, function (err, window) {
        if (err) { return tubesio.finish(err); }

        var $ = window.jQuery;

        $('table.track-grid-top-100 tr').each(function (i) {
            if (i === 0) { return; } // Skip header row

            var $row = $(this);
            
            result.push({
                position: parseInt($row.find('td:nth-child(1)').text()),
                trackName: $row.find('td:nth-child(4) > a').text(),
                artists: $row.find('td:nth-child(5) > a').text(),
                remixers: $row.find('td:nth-child(6) > a').text(), 
                label: $row.find('td:nth-child(7) > a').text(),
                genre: $row.find('td:nth-child(8) > a').text(),
                releaseDate: Date.parse($row.find('td:nth-child(9)').text())
            });
        });

        return tubesio.finish(result);
    });
}
1147875
def get_results():
    """"""
    # store info in a dictionary {name -> shortname}
    res = {}
    session = requests.Session()
    baseUrl = 'http://comicfury.com/search.php?search=1&webcomics=Search+for+webcomics&query=&worder=5&asc=1&incvi=1&incse=1&incnu=1&incla=1&all_ge=1&all_st=1&all_la=1&page='
    pages = 382
    for i in range(1, pages+1):
        url = baseUrl + str(i)
        handle_url(url, session, res)
    save_result(res, json_file)