Dispatches metrics streamed by collector
771140
def StreamMetrics(self, request_iterator, context):
        """"""
        LOG.debug("StreamMetrics called")

        # set up arguments
        collect_args = (next(request_iterator))
        max_metrics_buffer = 0
        max_collect_duration = 0
        cfg = Metric(pb=collect_args.Metrics_Arg.metrics[0])
        try:
            max_metrics_buffer = int(cfg.config["max-metrics-buffer"])
        except Exception as ex:
            LOG.debug("Unable to get schedule parameters: {}".format(ex))
        try:
            max_collect_duration = int(cfg.config["max-collect-duration"])
        except Exception as ex:
            LOG.debug("Unable to get schedule parameters: {}".format(ex))
        if max_metrics_buffer > 0:
            self.max_metrics_buffer = max_metrics_buffer
        if max_collect_duration > 0:
            self.max_collect_duration = max_collect_duration

        # start collection thread
        thread = threading.Thread(target=self._stream_wrapper, args=(collect_args,),)
        thread.daemon = True
        thread.start()

        # stream metrics
        metrics = []
        metrics_to_stream = []
        stream_timeout = self.max_collect_duration
        while context.is_active():
            try:
                # wait for metrics until timeout is reached
                t_start = time.time()
                metrics = self.metrics_queue.get(block=True, timeout=stream_timeout)
                elapsed = round(time.time() - t_start)
                stream_timeout -= elapsed
            except queue.Empty:
                LOG.debug("Max collect duration exceeded. Streaming {} metrics".format(len(metrics_to_stream)))
                metrics_col = CollectReply(Metrics_Reply=MetricsReply(metrics=[m.pb for m in metrics_to_stream]))
                metrics_to_stream = []
                stream_timeout = self.max_collect_duration
                yield metrics_col
            else:
                for metric in metrics:
                    metrics_to_stream.append(metric)
                    if len(metrics_to_stream) == self.max_metrics_buffer:
                        LOG.debug("Max metrics buffer reached. Streaming {} metrics".format(len(metrics_to_stream)))
                        metrics_col = CollectReply(
                            Metrics_Reply=MetricsReply(metrics=[m.pb for m in metrics_to_stream]))
                        metrics_to_stream = []
                        stream_timeout = self.max_collect_duration
                        yield metrics_col
                # stream metrics if max_metrics_buffer is 0 or enough metrics has been collected
                if self.max_metrics_buffer == 0:
                    LOG.debug("Max metrics buffer set to 0. Streaming {} metrics".format(len(metrics_to_stream)))
                    metrics_col = CollectReply(Metrics_Reply=MetricsReply(metrics=[m.pb for m in metrics_to_stream]))
                    metrics_to_stream = []
                    stream_timeout = self.max_collect_duration
                    yield metrics_col

        # sent notification if stream has been stopped
        self.done_queue.put(True)
312390
def Process(self, request, context):
        """"""
        LOG.debug("Process called")
        try:
            metrics = self.plugin.process(
                [Metric(pb=m) for m in request.Metrics],
                ConfigMap(pb=request.Config)
            )
            return MetricsReply(metrics=[m.pb for m in metrics])
        except Exception as err:
            msg = "message: {}\n\nstack trace: {}".format(
                err, traceback.format_exc())
            return MetricsReply(metrics=[], error=msg)
235095
function (metric) {
      metric.workerId = workerId
      metric.pid = process.pid
      if (!metric.sct) {
        if (metric.name && /collectd/.test(metric.name)) {
          metric.sct = 'OS'
        } else {
          metric.sct = 'APP'
        }
      }
      if (!metric.ts) {
        metric.ts = new Date().getTime()
      }
      if (!metric.type) {
        metric.type = this.defaultType
      }
      if (!metric.filters && metric.type === 'njs') {
        metric.filters = this.defaultFilters()
      }
      metric.spmLine = this.formatLine(metric)
      this.emit('metric', metric)
      if (metric.name) {
        this.emit(metric.name, metric)
      }
    }
864961
def process(self, metric):
        
        path = metric.getCollectorPath()
        path += '.'
        path += metric.getMetricPath()
        if self.config['apply_metric_prefix']:
            path = metric.getPathPrefix() + '.' + path

        if self.include_reg.match(path):
            if metric.metric_type == 'GAUGE':
                m_type = 'gauge'
            else:
                m_type = 'counter'
            self.queue.add(path,                # name
                           float(metric.value),  # value
                           type=m_type,
                           source=metric.host,
                           measure_time=metric.timestamp)
            self.current_n_measurements += 1
        else:
            self.log.debug("LibratoHandler: Skip %s, no include_filters match",
                           path)

        if (self.current_n_measurements >= self.queue_max_size or
                time.time() >= self.queue_max_timestamp):
            self.log.debug("LibratoHandler: Sending batch size: %d",
                           self.current_n_measurements)
            self._send()
181886
def expose_endpoint(self):
        
        @self._app.route('/metrics', methods=['GET'])
        async def expose_metrics(request):
            return raw(self._get_metrics_data(),
                       content_type=CONTENT_TYPE_LATEST)
723128
async def push_stats(self):
        """"""
        snapshot = self._make_stats()
        try:
            serialized = json.dumps(snapshot)
            await self._call_redis(aioredis.Redis.set, state.MANAGER_LISTENER_STATS, serialized)
            await self._call_redis(aioredis.Redis.expire, state.MANAGER_LISTENER_STATS, 3600)
        except TypeError:
            logger.error(__(
                "Listener can't serialize statistics:\n\n{}",
                traceback.format_exc()
            ))
        except aioredis.RedisError:
            logger.error(__(
                "Listener can't store updated statistics:\n\n{}",
                traceback.format_exc()
            ))
710749
def _collect_metrics_async(self, instance, query_specs):
        
        # ## <TEST-INSTRUMENTATION>
        t = Timer()
        # ## </TEST-INSTRUMENTATION>
        i_key = self._instance_key(instance)
        server_instance = self._get_server_instance(instance)
        perfManager = server_instance.content.perfManager
        custom_tags = instance.get('tags', [])
        results = perfManager.QueryPerf(query_specs)
        if results:
            for mor_perfs in results:
                mor_name = str(mor_perfs.entity)
                try:
                    mor = self.mor_cache.get_mor(i_key, mor_name)
                except MorNotFoundError:
                    self.log.error(
                        "Trying to get metrics from object %s deleted from the cache, skipping. "
                        "Consider increasing the parameter `clean_morlist_interval` to avoid that",
                        mor_name,
                    )
                    continue

                for result in mor_perfs.value:
                    counter_id = result.id.counterId
                    if not self.metadata_cache.contains(i_key, counter_id):
                        self.log.debug(
                            "Skipping value for counter {}, because there is no metadata about it".format(
                                ensure_unicode(counter_id)
                            )
                        )
                        continue

                    # Metric types are absolute, delta, and rate
                    metric_name = self.metadata_cache.get_metadata(i_key, result.id.counterId).get('name')

                    if self.in_compatibility_mode(instance):
                        if metric_name not in ALL_METRICS:
                            self.log.debug("Skipping unknown `{}` metric.".format(ensure_unicode(metric_name)))
                            continue

                    if not result.value:
                        self.log.debug(
                            "Skipping `{}` metric because the value is empty".format(ensure_unicode(metric_name))
                        )
                        continue

                    instance_name = result.id.instance or "none"
                    value = self._transform_value(instance, result.id.counterId, result.value[0])

                    tags = ['instance:{}'.format(ensure_unicode(instance_name))]
                    if not mor['hostname']:  # no host tags available
                        tags.extend(mor['tags'])

                    # vsphere "rates" should be submitted as gauges (rate is
                    # precomputed).
                    self.gauge(
                        "vsphere.{}".format(ensure_unicode(metric_name)),
                        value,
                        hostname=mor['hostname'],
                        tags=tags + custom_tags,
                    )

        # ## <TEST-INSTRUMENTATION>
        self.histogram('datadog.agent.vsphere.metric_colection.time', t.total(), tags=custom_tags)
853083
def StreamInChunks(self, callback=None, finish_callback=None,
                       additional_headers=None):
        """"""
        return self.__StreamMedia(
            callback=callback, finish_callback=finish_callback,
            additional_headers=additional_headers)
624035
function proxyStream(err, stats) {
    return through.obj(function(chunk, enc, cb) {
        processStats.call(this, chunk, stats);

        if (_.isError(err)) {
            this.emit('error', wrapError(err));
        }

        cb(null, chunk);
    });
}
973537
def _handle_streamer_finished(self, index, succeeded, highest_ack):
        """"""

        self._logger.debug("Rolling back streamer %d after streaming, highest ack from streaming subsystem was %d", index, highest_ack)
        self.acknowledge_streamer(index, highest_ack, False)
265246
def statsd_middleware_factory(app, handler):
    """"""
    @coroutine
    def middleware(request):
        """Send stats to statsd."""
        timer = Timer()
        timer.start()

        statsd = yield from app.ps.metrics.client()
        pipe = statsd.pipe()
        pipe.incr('request.method.%s' % request.method)

        try:
            response = yield from handler(request)
            pipe.incr('response.status.%s' % response.status)
            return response

        except HTTPException as exc:
            pipe.incr('response.status.%s' % exc.status_code)
            raise

        except Exception:
            pipe.incr('response.exception')
            raise

        finally:
            timer.stop()
            pipe.timing('response.time', timer.ms)
            pipe.disconnect()

    return middleware
1201579
def StreamInChunks(self, callback=None, finish_callback=None,
                       additional_headers=None):
        """"""
        self.StreamMedia(callback=callback, finish_callback=finish_callback,
                         additional_headers=additional_headers,
                         use_chunks=True)
954843
private void collectPoolInfoMetrics() {
    Map<PoolInfo, PoolInfoMetrics> newPoolNameToMetrics = new HashMap<PoolInfo,
      PoolInfoMetrics>();
    long now = ClusterManager.clock.getTime();

    Map<PoolInfo, Long> poolInfoAverageFirstWaitMs =
        sessionManager.getTypePoolInfoAveFirstWaitMs(type);

    // The gets + puts below are OK because only one thread is doing it.
    for (PoolGroupSchedulable poolGroup : poolGroupManager.getPoolGroups()) {
      int poolGroupSessions = 0;
      for (PoolSchedulable pool : poolGroup.getPools()) {
        MetricsRecord poolRecord =
            poolInfoToMetricsRecord.get(pool.getPoolInfo());
        if (poolRecord == null) {
          poolRecord = metrics.getContext().createRecord(
              "pool-" + pool.getName());
          poolInfoToMetricsRecord.put(pool.getPoolInfo(), poolRecord);
        }

        PoolInfoMetrics poolMetrics = new PoolInfoMetrics(pool.getPoolInfo(),
            type, poolRecord);
        poolMetrics.setCounter(
            MetricName.GRANTED, pool.getGranted());
        poolMetrics.setCounter(
            MetricName.REQUESTED, pool.getRequested());
        poolMetrics.setCounter(
            MetricName.SHARE, (long) pool.getShare());
        poolMetrics.setCounter(
            MetricName.MIN, pool.getMinimum());
        poolMetrics.setCounter(
            MetricName.MAX, pool.getMaximum());
        poolMetrics.setCounter(
            MetricName.WEIGHT, (long) pool.getWeight());
        poolMetrics.setCounter(
            MetricName.SESSIONS, pool.getScheduleQueue().size());
        poolMetrics.setCounter(
            MetricName.STARVING, pool.getStarvingTime(now) / 1000);
        Long averageFirstTypeMs =
            poolInfoAverageFirstWaitMs.get(pool.getPoolInfo());
        poolMetrics.setCounter(MetricName.AVE_FIRST_WAIT_MS,
            (averageFirstTypeMs == null) ?
                0 : averageFirstTypeMs.longValue());

        newPoolNameToMetrics.put(pool.getPoolInfo(), poolMetrics);
        poolGroupSessions += pool.getScheduleQueue().size();
      }

      MetricsRecord poolGroupRecord =
          poolInfoToMetricsRecord.get(poolGroup.getName());
      if (poolGroupRecord == null) {
        poolGroupRecord = metrics.getContext().createRecord(
            "poolgroup-" + poolGroup.getName());
        poolInfoToMetricsRecord.put(poolGroup.getPoolInfo(), poolGroupRecord);
      }

      PoolInfoMetrics poolGroupMetrics =
          new PoolInfoMetrics(poolGroup.getPoolInfo(), type, poolGroupRecord);
      poolGroupMetrics.setCounter(
          MetricName.GRANTED, poolGroup.getGranted());
      poolGroupMetrics.setCounter(
          MetricName.REQUESTED, poolGroup.getRequested());
      poolGroupMetrics.setCounter(
          MetricName.SHARE, (long) poolGroup.getShare());
      poolGroupMetrics.setCounter(
          MetricName.MIN, poolGroup.getMinimum());
      poolGroupMetrics.setCounter(
          MetricName.MAX, poolGroup.getMaximum());
      poolGroupMetrics.setCounter(
          MetricName.SESSIONS, poolGroupSessions);
      newPoolNameToMetrics.put(poolGroup.getPoolInfo(), poolGroupMetrics);
    }

    poolInfoToMetrics = newPoolNameToMetrics;
  }
648551
def Run(self, args):
    """"""
    # Make sure we limit the size of our output
    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:
      raise RuntimeError("Can not read buffers this large.")

    data = vfs.ReadVFS(
        args.pathspec,
        args.offset,
        args.length,
        progress_callback=self.Progress)
    result = rdf_protodict.DataBlob(
        data=zlib.compress(data),
        compression=rdf_protodict.DataBlob.CompressionType.ZCOMPRESSION)

    digest = hashlib.sha256(data).digest()

    # Ensure that the buffer is counted against this response. Check network
    # send limit.
    self.ChargeBytesToSession(len(data))

    # Now return the data to the server into the special TransferStore well
    # known flow.
    self.grr_worker.SendReply(
        result, session_id=rdfvalue.SessionID(flow_name="TransferStore"))

    # Now report the hash of this blob to our flow as well as the offset and
    # length.
    self.SendReply(
        rdf_client.BufferReference(
            offset=args.offset, length=len(data), data=digest))
1070920
def stream_request(self, request, out_future):
        """"""
        request.close_argstreams()

        def on_done(future):
            if future.exception() and out_future.running():
                out_future.set_exc_info(future.exc_info())
            request.close_argstreams(force=True)

        stream_future = self._stream(request, self.request_message_factory)
        stream_future.add_done_callback(on_done)
        return stream_future
1099556
@Override
  public void getMetrics(MetricsCollector collector, boolean all) {
    MetricsRecordBuilder metricsRecordBuilder = collector.addRecord("sparkShuffleService");

    for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
      collectMetric(metricsRecordBuilder, entry.getKey(), entry.getValue());
    }
  }
245064
def process_streamers(self):
        """"""

        # Check for any triggered streamers and pass them to stream manager
        in_progress = self._stream_manager.in_progress()
        triggered = self.graph.check_streamers(blacklist=in_progress)

        for streamer in triggered:
            self._stream_manager.process_streamer(streamer, callback=self._handle_streamer_finished)