Wrapper to assist with paginated responses from Digicert's REST API.
760088
def _paginate(url, topkey, *args, **kwargs):
    '''
    
    '''

    ret = salt.utils.http.query(url, **kwargs)
    if 'errors' in ret['dict']:
        return ret['dict']

    lim = int(ret['dict']['page']['limit'])
    total = int(ret['dict']['page']['total'])

    if total == 0:
        return {}

    numpages = (total / lim) + 1

    # If the count returned is less than the page size, just return the dict
    if numpages == 1:
        return ret['dict'][topkey]

    aggregate_ret = ret['dict'][topkey]
    url = args[0]
    for p in range(2, numpages):
        param_url = url + '?offset={0}'.format(lim * (p - 1))
        next_ret = salt.utils.http.query(param_url, kwargs)
        aggregate_ret[topkey].extend(next_ret['dict'][topkey])

    return aggregate_ret
572225
def fetch_pages(response, limit, type)
      pages = [GoogleApps::Atom.feed(response.body)]

      while (pages.last.next_page) and (pages.count * PAGE_SIZE[:user] < limit)
        pages << get_next_page(pages.last.next_page, type)
      end
      pages
    end
1144836
def api_get_collection(path, collection, *args)
      Enumerator.new do |enum|
        args[0] = {} unless args[0]
        page_size = (args[0].delete(:page_size) { 100 }).to_i
        page_number = (args[0].delete(:page_number) { 1 }).to_i

        loop do
          uri = URI::HTTP.build(path: "/#{path}", query: URI.encode_www_form(pageSize: page_size, pageNumber: page_number)).request_uri

          res = api_get(uri, *args)
          raise TableauError, res if res.code.to_s != '200'

          # ensure the result is an array because it will not be an array if there is only one element
          [collection.split('.').reduce(res['tsResponse']) { |acc, elem| acc && acc[elem] }].flatten.compact.each do |obj|
            enum.yield obj
          end

          break if res['tsResponse']['pagination'].nil?
          break if page_number >= (res['tsResponse']['pagination']['totalAvailable'].to_i / page_size.to_f).ceil

          page_number += 1
        end
      end
    end
986278
def fromapi(_class, apiresponse):
        """"""
        for resp in apiresponse['sb']:
            # Extract details from dict
            _id = "n/a" or resp.get("nm")
            subject = resp.get("sbj")
            text = resp.get('dtl') + "\n" + resp.get('brf')
            priority = "n/a" or resp.get('prty')
            for_stops, for_routes = [], []
            svc = resp.get('srvc')
        
            # Create list of affected routes/stops, if there are any
            if svc:
                has_stop = 'stpid' in svc or 'stpnm' in svc
                has_rt = 'rt' in svc or 'rtdir' in svc
            
                if has_stop: 
                    aff = _class.affected_service('stop', svc.get('stpid'), svc.get('stpnm'))
                    for_stops.append(aff)
                if has_rt:
                    aff = _class.affected_service('route', svc.get('rt'), svc.get('rtdir'))
                    for_routes.append(aff)

            yield _class(_id, subject, text, priority, for_stops, for_routes)
196103
def analyze_url_list(self, response):
        
        response = json.loads(response.text)
        for _, block in response['blocks'].items():
            for attribute in block:
                parsed = urlparse(block[attribute])
                # find urls in the JSON response
                if parsed.scheme and parsed.netloc:
                    yield self.make_requests_from_url(block[attribute])
440618
def each(path, params = {}, header = {}, &block)
      # retrieve the resources using the max page size (least nr of API calls)
      next_path = expand_path(path, {per_page: MAX_PAGE_SIZE, page: 1}.merge(params))
      size = 0
      while next_path
        # retrieve the records (with retry and optionally wait for rate-limit)
        response = get(next_path, {}, header)
        # raise exception in case the response is invalid
        raise ::Itrp::Exception.new(response.message) unless response.valid?
        # yield the resources
        response.json.each{ |resource| yield resource }
        size += response.json.size
        # go to the next page
        next_path = response.pagination_relative_link(:next)
      end
      size
    end
985472
def handle_pagination(self, page_num=None, page_size=None):
        
        self._response_json = self.get_next_page(page_num=page_num, page_size=page_size)
        self.update_attrs()
        self.position = 0
        self.values = self.process_page()
1002339
def paginate(self, request, offset=0, limit=None):
        """"""
        return self.collection.offset(offset).limit(limit), self.collection.count()
736209
def resource_collection_response(cls, offset=0, limit=20):
        
        request_args = {'page[offset]': offset, 'page[limit]': limit}
        return cls.get_collection(request_args)
830757
def paginated_retrieval(methodname, itemtype):
    """"""
    return compose(
        reusable,
        basic_interaction,
        map_yield(partial(_params_as_get, methodname)),
    )
904203
def ReadRequestsAndResponses(self,
                               session_id,
                               timestamp=None,
                               request_limit=None,
                               response_limit=None):
    """"""
    subject = session_id.Add("state")
    requests = {}

    # Get some requests.
    for predicate, serialized, _ in self.ResolvePrefix(
        subject,
        self.FLOW_REQUEST_PREFIX,
        limit=request_limit,
        timestamp=timestamp):

      request_id = predicate.split(":", 1)[1]
      requests[str(subject.Add(request_id))] = serialized

    # And the responses for them.
    response_data = dict(
        self.MultiResolvePrefix(
            list(iterkeys(requests)),
            self.FLOW_RESPONSE_PREFIX,
            limit=response_limit,
            timestamp=timestamp))

    for urn, request_data in sorted(iteritems(requests)):
      request = rdf_flow_runner.RequestState.FromSerializedString(request_data)
      responses = []
      for _, serialized, timestamp in response_data.get(urn, []):
        msg = rdf_flows.GrrMessage.FromSerializedString(serialized)
        msg.timestamp = timestamp
        responses.append(msg)

      yield (request, sorted(responses, key=lambda msg: msg.response_id))
1059514
def _get_next_page_response(self):
        
        params = self._get_query_params()
        if self._page_size is not None:
            params["maxResults"] = self._page_size
        return self.api_request(
            method=self._HTTP_METHOD, path=self.path, query_params=params
        )
1176976
def search(self, query, pagination, result_field):
        
        result = []
        url = "/".join((self.url, query))
        while url:
            log.debug("Pagure query: {0}".format(url))
            try:
                response = requests.get(url, headers=self.headers)
                log.data("Response headers:\n{0}".format(response.headers))
            except requests.RequestException as error:
                log.error(error)
                raise ReportError("Pagure search {0} failed.".format(self.url))
            data = response.json()
            objects = data[result_field]
            log.debug("Result: {0} fetched".format(
                listed(len(objects), "item")))
            log.data(pretty(data))
            # FIXME later: Work around https://pagure.io/pagure/issue/4057
            if not objects:
                break
            result.extend(objects)
            url = data[pagination]['next']
        return result
1118590
def _iterator(self, response):
        '''
        
        '''

        while True:
            result = deque(self._parse_data(response))
            del response
            if result:
                doc_count = len(result)
                last = result.pop()
                while result:
                    yield result.popleft()

                # We expect doc_count = self._page_size + 1 results, if
                # we have self._page_size or less it means we are on the
                # last page and need to return the last result.
                if doc_count < self._real_page_size:
                    yield last
                    break
                del result

                # if we are in a view, keys could be duplicate so we
                # need to start from the right docid
                if last['id']:
                    response = self._call(startkey=last['key'],
                                          startkey_docid=last['id'])
                # reduce result keys are unique by definition
                else:
                    response = self._call(startkey=last['key'])

            else:
                break
693980
def get_collection(self, request, **kwargs):
        
        qs = self.queryset(request, **kwargs)

        if self.display_collection_fields:
            display_fields = self.display_collection_fields
        else:
            display_fields = self.display_fields

        if self.paginate_by is not None:
            page = request.GET.get('page', 1)
            paginator = Paginator(qs, self.paginate_by)
            try:
                qs = paginator.page(page).object_list
            except PageNotAnInteger:
                data = _('Invalid `page` parameter: Not a valid integer.')
                return HttpResponseBadRequest(data)
            except EmptyPage:
                data = _('Invalid `page` parameter: Out of range.')
                return HttpResponseBadRequest(data)
        data = [
            self.serialize(obj, ['id'] + list(display_fields)) for obj in qs
        ]
        return HttpResponse(self.json_dumps(data), content_type='application/json')
422731
def compat_get_paginated_response(view, page):
    
    if DRFVLIST[0] == 3 and DRFVLIST[1] >= 1:
        from rest_messaging.serializers import ComplexMessageSerializer  # circular import
        serializer = ComplexMessageSerializer(page, many=True)
        return view.get_paginated_response(serializer.data)
    else:
        serializer = view.get_pagination_serializer(page)
        return Response(serializer.data)
153388
def _get_responses(cls, requests, dispatcher):
        
        for request in requests:
            def response(**kwargs):
                return cls.RESPONSE_CLASS_MAP[request.JSONRPC_VERSION](
                    _id=request._id, **kwargs)

            try:
                method = dispatcher[request.method]
            except KeyError:
                output = response(error=JSONRPCMethodNotFound()._data)
            else:
                try:
                    result = method(*request.args, **request.kwargs)
                except JSONRPCDispatchException as e:
                    output = response(error=e.error._data)
                except Exception as e:
                    data = {
                        "type": e.__class__.__name__,
                        "args": e.args,
                        "message": str(e),
                    }
                    if isinstance(e, TypeError) and is_invalid_params(
                            method, *request.args, **request.kwargs):
                        output = response(
                            error=JSONRPCInvalidParams(data=data)._data)
                    else:
                        # logger.exception("API Exception: {0}".format(data))
                        print("API Exception: {0}".format(data))
                        output = response(
                            error=JSONRPCServerError(data=data)._data)
                else:
                    output = response(result=result)
            finally:
                if not request.is_notification:
                    yield output