Send a GET/POST request or get data from cache, retry if it fails, and return a tuple of store in cache callback, response content.
519552
async def query(self, url, *, post_data=None, headers=None, verify=True, cache=None, pre_cache_callback=None):
    
    async def store_in_cache_callback():
      pass
    if cache is not None:
      # try from cache first
      if post_data is not None:
        if (url, post_data) in cache:
          self.logger.debug("Got data for URL '%s' %s from cache" % (url, dict(post_data)))
          return store_in_cache_callback, cache[(url, post_data)]
      elif url in cache:
        self.logger.debug("Got data for URL '%s' from cache" % (url))
        return store_in_cache_callback, cache[url]

    domain_rate_watcher = rate_watcher.AccessRateWatcher(self.watcher_db_filepath,
                                                         url,
                                                         self.min_delay_between_accesses,
                                                         jitter_range_ms=self.jitter_range_ms,
                                                         logger=self.logger)

    for attempt, time_to_sleep in enumerate(redo.retrier(max_attempts=HTTP_MAX_ATTEMPTS,
                                                         sleeptime=1,
                                                         max_sleeptime=HTTP_MAX_RETRY_SLEEP_S,
                                                         sleepscale=1.5),
                                            1):
      await domain_rate_watcher.waitAccessAsync()

      try:
        if post_data is not None:
          async with self.session.post(url,
                                       data=post_data,
                                       headers=self._buildHeaders(headers),
                                       timeout=HTTP_NORMAL_TIMEOUT,
                                       ssl=verify) as response:
            content = await response.read()
        else:
          async with self.session.get(url,
                                      headers=self._buildHeaders(headers),
                                      timeout=HTTP_NORMAL_TIMEOUT,
                                      ssl=verify) as response:
            content = await response.read()

        if cache is not None:
          async def store_in_cache_callback():
            if pre_cache_callback is not None:
              # process
              try:
                data = await pre_cache_callback(content)
              except Exception:
                data = content
            else:
              data = content

            # add to cache
            if post_data is not None:
              cache[(url, post_data)] = data
            else:
              cache[url] = data

      except (asyncio.TimeoutError, aiohttp.ClientError) as e:
        self.logger.warning("Querying '%s' failed (attempt %u/%u): %s %s" % (url,
                                                                             attempt,
                                                                             HTTP_MAX_ATTEMPTS,
                                                                             e.__class__.__qualname__,
                                                                             e))
        if attempt == HTTP_MAX_ATTEMPTS:
          raise
        else:
          self.logger.debug("Retrying in %.3fs" % (time_to_sleep))
          await asyncio.sleep(time_to_sleep)

      else:
        break  # http retry loop

    response.raise_for_status()

    return store_in_cache_callback, content
521298
def execute(self, ignore_cache=False, raise_on_error=True):
        
        if ignore_cache or not hasattr(self, '_response'):
            es = connections.get_connection(self._using)

            responses = es.msearch(
                index=self._index,
                body=self.to_dict(),
                **self._params
            )

            out = []
            for s, r in zip(self._searches, responses['responses']):
                if r.get('error', False):
                    if raise_on_error:
                        raise TransportError('N/A', r['error']['type'], r['error'])
                    r = None
                else:
                    r = Response(s, r)
                out.append(r)

            self._response = out

        return self._response
889939
def _request(self, url, method='GET', params=None):
        """"""
        self.connected = True
        retry_counter = 0

        method = method.lower()
        func = getattr(self.client, method)
        params, _ = _transparent_params(params)

        def _send(retry_counter):
            requests_args = {}
            for k, v in self.client_args.items():
                # Maybe this should be set as a class
                # variable and only done once?
                if k in ('timeout', 'allow_redirects', 'verify'):
                    requests_args[k] = v

            while self.connected:
                try:
                    if method == 'get':
                        requests_args['params'] = params
                    else:
                        requests_args['data'] = params

                    response = func(url, **requests_args)
                except requests.exceptions.Timeout:
                    self.on_timeout()
                else:
                    if response.status_code != 200:
                        self.on_error(response.status_code, response.content)

                    if self.retry_count and \
                       (self.retry_count - retry_counter) > 0:
                        time.sleep(self.retry_in)
                        retry_counter += 1
                        _send(retry_counter)

                    return response

        while self.connected:
            response = _send(retry_counter)

            for line in response.iter_lines(self.chunk_size):
                if not self.connected:
                    break
                if line:
                    try:
                        if is_py3:
                            line = line.decode('utf-8')
                        data = json.loads(line)
                    except ValueError:  # pragma: no cover
                        self.on_error(response.status_code,
                                      'Unable to decode response, \
                                      not valid JSON.')
                    else:
                        if self.on_success(data):  # pragma: no cover
                            for message_type in self.handlers:
                                if message_type in data:
                                    handler = getattr(self,
                                                      'on_' + message_type,
                                                      None)
                                    if handler \
                                       and callable(handler) \
                                       and not handler(data.get(message_type)):
                                        break

        response.close()
757967
def execute(self, ignore_cache=False):
        
        if ignore_cache or not hasattr(self, '_response'):
            es = connections.get_connection(self._using)

            self._response = self._response_class(
                self,
                es.search(
                    index=self._index,
                    body=self.to_dict(),
                    **self._params
                )
            )
        return self._response
1256548
def retrieve(self, request):
        
        if request.method not in CACHE_VERBS:
            return
        try:
            response = self.data[request.url]['response']
            logger.info('Retrieved response from cache')
            return response
        except KeyError:
            return None
17307
def request(self, server=None):
        
        request = {
            "headers": {"Content-Type": "application/json"},
            "timeout": self._pump.timeout,
            "data": self.context,
        }

        url = "{proto}://{server}/{endpoint}".format(
            proto=self._pump.protocol,
            server=server or self.server,
            endpoint=self.ENDPOINT,
        )

        response = self._pump._requester(requests.post, url, **request)

        try:
            server_data = response.json()
        except ValueError:
            raise ClientException(response.content)

        if "error" in server_data:
            raise ClientException(server_data["error"], self.context)

        _log.debug("Client registration recieved: %(id)s %(secret)s %(expire)s", {
            "id": server_data["client_id"],
            "secret": server_data["client_secret"],
            "expire": server_data["expires_at"],
        })

        return server_data
11150
def _make_request_to_server(self, query_function, raise_for_status=True,
                                time_limit_seconds=2, retry_delay_seconds=0.2):
        
        start_time = datetime.datetime.now()
        while datetime.datetime.now() - start_time < datetime.timedelta(
                0, time_limit_seconds):
            error = None
            response = None
            try:
                response = query_function()
            except requests.exceptions.ConnectionError as e:
                error = ServerConnectionError(
                    "No response from server.\n%s" % e)
            except:
                if response:
                    logger.info(response.text)
                raise
            if response is not None and raise_for_status:
                # raises requests.exceptions.HTTPError
                self._raise_for_status(response)
            if error:
                time.sleep(retry_delay_seconds)
                continue
            else:
                return response
        raise error
1123042
function makeRequest () {
    request = self.request.apply(null, args);
    requestMiddleware.use(request);
    request.on("response", function (response) {
      var contentEncoding, gzipper;

      //Only cache successful responses
      if (response.statusCode >= 200 && response.statusCode < 300) {
        response.on('error', function (error) {
          self.handleError(error);
        });
        fs.writeFile(self.cacheDirectory + key + ".json", JSON.stringify(response.headers), function (error) {
          if (error) self.handleError(error);
        });

        responseWriter = fs.createWriteStream(self.cacheDirectory + key);

        responseWriter.on('error', function (error) {
          self.handleError(error);
        });

        contentEncoding = response.headers['content-encoding'] || '';
        contentEncoding = contentEncoding.trim().toLowerCase();

        if (contentEncoding === 'gzip') {
          response.on('error', function (error) {
            responseWriter.end();
          });
          response.pipe(responseWriter);
        } else {
          gzipper = zlib.createGzip();
          response.on('error', function (error) {
            gzipper.end();
          });
          gzipper.on('error', function (error) {
            self.handleError(error);
            responseWriter.end();
          });
          responseWriter.on('error', function (error) {
            response.unpipe(gzipper);
            gzipper.end();
          });
          response.pipe(gzipper).pipe(responseWriter);
        }
      }
    });
    self.emit("request", args[0]);
  }
1286042
def _handle_response(self):
        
        content = self.cache['restbase']['info']['content-type']
        if content.startswith('text/html'):
            html = self.cache['restbase']['response']
            if isinstance(html, bytes):
                html = html.decode('utf-8')
            self.data['html'] = html
            return

        response = self._load_response('restbase')

        http_status = self.cache['restbase']['info']['status']
        if http_status == 404:
            raise LookupError(self.cache['restbase']['query'])

        if self.params.get('endpoint') == '/page/':
            msg = "RESTBase /page/ entry points: %s" % response.get('items')
            utils.stderr(msg)
            del self.cache['restbase']
            return

        return response
775746
function() {
        // Send request
        var sendRequest = function() {
          debug('Calling: %s, retry: %s', entry.name, attempts - 1);
          // Make request and handle response or error
          return makeRequest(
            that,
            entry.method,
            url,
            payload,
            query
          ).then(function(res) {
            // If request was successful, accept the result
            debug('Success calling: %s, (%s retries)',
              entry.name, attempts - 1);
            if (monitor) {
              var d = process.hrtime(start);
              monitor.measure([entry.name, 'success'], d[0] * 1000 + d[1] / 1000000);
              monitor.count([entry.name, 'success']);
            }
            if (!_.includes(res.headers['content-type'], 'application/json') || !res.body) {
              debug('Empty response from server: call: %s, method: %s', entry.name, entry.method);
              return undefined;
            }
            return res.body;
          }, function(err) {
            // If we got a response we read the error code from the response
            var res = err.response;
            if (res) {
              // Decide if we should retry
              if (attempts <= that._options.retries &&
                  res.status >= 500 &&  // Check if it's a 5xx error
                  res.status < 600) {
                debug('Error calling: %s now retrying, info: %j',
                  entry.name, res.body);
                return retryRequest();
              }
              // If not retrying, construct error object and reject
              debug('Error calling: %s NOT retrying!, info: %j',
                entry.name, res.body);
              var message = 'Unknown Server Error';
              if (res.status === 401) {
                message = 'Authentication Error';
              }
              if (res.status === 500) {
                message = 'Internal Server Error';
              }
              err = new Error(res.body.message || message);
              err.body = res.body;
              err.code = res.body.code || 'UnknownError';
              err.statusCode = res.status;
              if (monitor) {
                var d = process.hrtime(start);

                var state = 'client-error';
                if (res.statusCode >= 500) {
                  state = 'server-error';
                }
                monitor.measure([entry.name, state], d[0] * 1000 + d[1] / 1000000);
                monitor.count([entry.name, state]);
              }
              throw err;
            }

            // Decide if we should retry
            if (attempts <= that._options.retries) {
              debug('Request error calling %s (retrying), err: %s, JSON: %s',
                entry.name, err, err);
              return retryRequest();
            }
            debug('Request error calling %s NOT retrying!, err: %s, JSON: %s',
              entry.name, err, err);
            if (monitor) {
              var d = process.hrtime(start);
              monitor.measure([entry.name, 'connection-error'], d[0] * 1000 + d[1] / 1000000);
              monitor.count([entry.name, 'connection-error']);
            }
            throw err;
          });
        };

        // Increment attempt count, but track how many we had before.
        attempts += 1;

        // If this is the first retry, ie we haven't retried yet, we make the
        // request immediately
        if (attempts === 1) {
          return sendRequest();
        } else {
          var delay;
          // First request is attempt = 1, so attempt = 2 is the first retry
          // we subtract one to get exponents: 1, 2, 3, 4, 5, ...
          delay = Math.pow(2, attempts - 1) * that._options.delayFactor;
          // Apply randomization factor
          var rf = that._options.randomizationFactor;
          delay *= Math.random() * 2 * rf + 1 - rf;
          // Always limit with a maximum delay
          delay = Math.min(delay, that._options.maxDelay);
          // Sleep then send the request
          return new Promise(function(accept) {
            setTimeout(accept, delay);
          }).then(function() {
            return sendRequest();
          });
        }
      }
766839
def _req(self, endpoint, get_data=dict(), post_data=dict()):

        

        if get_data:
            request_parameter = "{}?{}".format(endpoint, urlencode(get_data))
        else:
            request_parameter = endpoint

        try:
            encdata = urlencode(post_data, True).encode('utf-8')
            response = self.oauth.request(request_parameter, data=encdata)
            self._checkResponseForErrors(response)
        except HTTPError as e:
            raise DeviantartError(e)

        return response
1246945
def response(cls, dic, status=200):
        
        response = JsonResponse(dic, status=status)
        response['Cache-Control'] = 'no-store'
        response['Pragma'] = 'no-cache'

        return response
99418
def post_request(self, url, params):
        

        data = encrypted_request(params)
        resp = self.session.post(url, data=data, timeout=self.timeout,
                                 proxies=self.proxies)
        result = resp.json()
        if result['code'] != 200:
            LOG.error('Return %s when try to post %s => %s',
                      result, url, params)
            raise PostRequestIllegal(result)
        else:
            return result
1357304
def process_response(self, request, response):
        """"""
        if not hasattr(request, '_cache_update_cache') or not request._cache_update_cache:
            # We don't need to update the cache, just return.
            return response
        if request.method != 'GET':
            # This is a stronger requirement than above. It is needed
            # because of interactions between this middleware and the
            # HTTPMiddleware, which throws the body of a HEAD-request
            # away before this middleware gets a chance to cache it.
            return response
        if not response.status_code == 200:
            return response
        # Try to get the timeout from the "max-age" section of the "Cache-
        # Control" header before reverting to using the default cache_timeout
        # length.
        timeout = get_max_age(response)
        if timeout == None:
            timeout = self.cache_timeout
        elif timeout == 0:
            # max-age was set to 0, don't bother caching.
            return response
        patch_response_headers(response, timeout)
        if timeout:
            cache_key = learn_cache_key(request, response, timeout, self.key_prefix)
            cache.set(cache_key, response, timeout)
            logging.debug("UpdateCacheMiddleware: setting %s -> %s params are: %s" % (cache_key, request.path, get_cache_key_parameters(request)))
        return response
1298179
def _handle_get(self, request_data):
        
        der = base64.b64decode(request_data)
        ocsp_request = self._parse_ocsp_request(der)
        return self._build_http_response(ocsp_request)
423607
def _request_api(self, **kwargs):
        
        _url = kwargs.get('url')
        _method = kwargs.get('method', 'GET')
        _status = kwargs.get('status', 200)

        counter = 0
        if _method not in ['GET', 'POST']:
            raise ValueError('Method is not GET or POST')

        while True:
            try:
                res = REQ[_method](_url, cookies=self._cookie)
                if res.status_code == _status:
                    break
                else:
                    raise BadStatusException(res.content)
            except requests.exceptions.BaseHTTPError:
                if counter < self._retries:
                    counter += 1
                    continue
                raise MaxRetryError
        self._last_result = res
        return res
1268696
async def retry_post(config, url, *args, **kwargs):
    
    return await _retry_do(config.session.post, url, *args,
                           **_make_headers(config, kwargs))