caculate total response time of all meta_datas
766389
def __get_total_response_time(meta_datas_expanded):
    
    try:
        response_time = 0
        for meta_data in meta_datas_expanded:
            response_time += meta_data["stat"]["response_time_ms"]

        return "{:.2f}".format(response_time)

    except TypeError:
        # failure exists
        return "N/A"
770020
def GetMetadata():
  """"""
  return [
      # GRR user-management metrics.
      stats_utils.CreateEventMetadata(
          "acl_check_time", fields=[("check_type", str)]),
      stats_utils.CreateCounterMetadata(
          "approval_searches",
          fields=[("reason_presence", str), ("source", str)]),

      # Cronjob metrics.
      stats_utils.CreateCounterMetadata("cron_internal_error"),
      stats_utils.CreateCounterMetadata(
          "cron_job_failure", fields=[("cron_job_id", str)]),
      stats_utils.CreateCounterMetadata(
          "cron_job_timeout", fields=[("cron_job_id", str)]),
      stats_utils.CreateEventMetadata(
          "cron_job_latency", fields=[("cron_job_id", str)]),

      # Access-control metrics.
      stats_utils.CreateCounterMetadata("grr_expired_tokens"),

      # Datastore metrics.
      stats_utils.CreateCounterMetadata("grr_commit_failure"),
      stats_utils.CreateCounterMetadata("datastore_retries"),
      stats_utils.CreateGaugeMetadata(
          "datastore_size",
          int,
          docstring="Size of data store in bytes",
          units="BYTES"),
      stats_utils.CreateCounterMetadata("grr_task_retransmission_count"),
      stats_utils.CreateCounterMetadata("grr_task_ttl_expired_count"),
      stats_utils.CreateEventMetadata(
          "db_request_latency",
          fields=[("call", str)],
          bins=[0.05 * 1.2**x for x in range(30)]),  # 50ms to ~10 secs
      stats_utils.CreateCounterMetadata(
          "db_request_errors", fields=[("call", str), ("type", str)]),

      # Threadpool metrics.
      stats_utils.CreateGaugeMetadata(
          "threadpool_outstanding_tasks", int, fields=[("pool_name", str)]),
      stats_utils.CreateGaugeMetadata(
          "threadpool_threads", int, fields=[("pool_name", str)]),
      stats_utils.CreateGaugeMetadata(
          "threadpool_cpu_use", float, fields=[("pool_name", str)]),
      stats_utils.CreateCounterMetadata(
          "threadpool_task_exceptions", fields=[("pool_name", str)]),
      stats_utils.CreateEventMetadata(
          "threadpool_working_time", fields=[("pool_name", str)]),
      stats_utils.CreateEventMetadata(
          "threadpool_queueing_time", fields=[("pool_name", str)]),

      # Worker and flow-related metrics.
      stats_utils.CreateCounterMetadata("grr_flows_stuck"),
      stats_utils.CreateCounterMetadata(
          "worker_bad_flow_objects", fields=[("type", str)]),
      stats_utils.CreateCounterMetadata(
          "worker_session_errors", fields=[("type", str)]),
      stats_utils.CreateCounterMetadata(
          "worker_flow_lock_error",
          docstring="Worker lock failures. We expect these to be high when the "
          "systemis idle."),
      stats_utils.CreateEventMetadata(
          "worker_flow_processing_time", fields=[("flow", str)]),
      stats_utils.CreateEventMetadata("worker_time_to_retrieve_notifications"),
      stats_utils.CreateCounterMetadata("grr_flow_completed_count"),
      stats_utils.CreateCounterMetadata("grr_flow_errors"),
      stats_utils.CreateCounterMetadata("grr_flow_invalid_flow_count"),
      stats_utils.CreateCounterMetadata("grr_request_retransmission_count"),
      stats_utils.CreateCounterMetadata("grr_response_out_of_order"),
      stats_utils.CreateCounterMetadata("grr_unique_clients"),
      stats_utils.CreateCounterMetadata("grr_worker_states_run"),
      stats_utils.CreateCounterMetadata("grr_well_known_flow_requests"),
      stats_utils.CreateCounterMetadata("flow_starts", fields=[("flow", str)]),
      stats_utils.CreateCounterMetadata("flow_errors", fields=[("flow", str)]),
      stats_utils.CreateCounterMetadata(
          "flow_completions", fields=[("flow", str)]),
      stats_utils.CreateCounterMetadata(
          "well_known_flow_requests", fields=[("flow", str)]),
      stats_utils.CreateCounterMetadata(
          "well_known_flow_errors", fields=[("flow", str)]),
      stats_utils.CreateEventMetadata("fleetspeak_last_ping_latency_millis"),

      # Hunt-related metrics.
      stats_utils.CreateCounterMetadata(
          "hunt_output_plugin_verifications", fields=[("status", str)]),
      stats_utils.CreateCounterMetadata(
          "hunt_output_plugin_verification_errors"),
      stats_utils.CreateCounterMetadata(
          "hunt_output_plugin_errors", fields=[("plugin", str)]),
      stats_utils.CreateCounterMetadata(
          "hunt_results_ran_through_plugin", fields=[("plugin", str)]),
      stats_utils.CreateCounterMetadata("hunt_results_compacted"),
      stats_utils.CreateCounterMetadata(
          "hunt_results_compaction_locking_errors"),
      stats_utils.CreateCounterMetadata("hunt_results_added"),

      # Metric used to identify the master in a distributed server setup.
      stats_utils.CreateGaugeMetadata("is_master", int),

      # GRR-API metrics.
      stats_utils.CreateEventMetadata(
          "api_method_latency",
          fields=[("method_name", str), ("protocol", str), ("status", str)]),
      stats_utils.CreateEventMetadata(
          "api_access_probe_latency",
          fields=[("method_name", str), ("protocol", str), ("status", str)]),

      # Client-related metrics.
      stats_utils.CreateCounterMetadata("grr_client_crashes"),
      stats_utils.CreateCounterMetadata(
          "client_pings_by_label", fields=[("label", str)]),

      # Metrics specific to GRR frontends.
      stats_utils.CreateGaugeMetadata(
          "frontend_active_count", int, fields=[("source", str)]),
      stats_utils.CreateGaugeMetadata("frontend_max_active_count", int),
      stats_utils.CreateCounterMetadata(
          "frontend_http_requests", fields=[("action", str), ("protocol",
                                                              str)]),
      stats_utils.CreateCounterMetadata(
          "frontend_in_bytes", fields=[("source", str)]),
      stats_utils.CreateCounterMetadata(
          "frontend_out_bytes", fields=[("source", str)]),
      stats_utils.CreateCounterMetadata(
          "frontend_request_count", fields=[("source", str)]),
      stats_utils.CreateCounterMetadata(
          "frontend_inactive_request_count", fields=[("source", str)]),
      stats_utils.CreateEventMetadata(
          "frontend_request_latency", fields=[("source", str)]),
      stats_utils.CreateEventMetadata("grr_frontendserver_handle_time"),
      stats_utils.CreateCounterMetadata("grr_frontendserver_handle_num"),
      stats_utils.CreateGaugeMetadata("grr_frontendserver_client_cache_size",
                                      int),
      stats_utils.CreateCounterMetadata("grr_messages_sent"),
      stats_utils.CreateCounterMetadata(
          "grr_pub_key_cache", fields=[("type", str)]),
  ]
630436
def metrics(self):
        
        masterThrp, backupThrp = self.getThroughputs(self.instances.masterId)
        r = self.instance_throughput_ratio(self.instances.masterId)
        m = [
            ("{} Monitor metrics:".format(self), None),
            ("Delta", self.Delta),
            ("Lambda", self.Lambda),
            ("Omega", self.Omega),
            ("instances started", self.instances.started),
            ("ordered request counts",
             {i: r[0] for i, r in self.numOrderedRequests.items()}),
            ("ordered request durations",
             {i: r[1] for i, r in self.numOrderedRequests.items()}),
            ("master request latencies", self.masterReqLatencies),
            ("client avg request latencies", {i: self.getLatency(i)
                                              for i in self.instances.ids}),
            ("throughput", {i: self.getThroughput(i)
                            for i in self.instances.ids}),
            ("master throughput", masterThrp),
            ("total requests", self.totalRequests),
            ("avg backup throughput", backupThrp),
            ("master throughput ratio", r)]
        return m
1326750
def per_version_data(self):
        
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            if len(data['by_version']) == 0:
                data['by_version'] = {'other': 0}
            ret[cache_date] = data['by_version']
        return ret
661665
def ReadClientStats(self, client_id,
                      min_timestamp,
                      max_timestamp
                     ):
    """"""
    results = []
    for timestamp, stats in iteritems(self.client_stats[client_id]):
      if min_timestamp <= timestamp <= max_timestamp:
        results.append(stats)
    return results
1141086
def _process_meta(self, response):
        """"""
        for key in self.META_ENVELOPES:
            self.meta[key] = response.get(key)
352763
def metas(self, prefix=None, limit=None, delimiter=None):
        
        limit = coalesce(limit, TOO_MANY_KEYS)
        keys = self.bucket.list(prefix=prefix, delimiter=delimiter)
        prefix_len = len(prefix)
        output = []
        for i, k in enumerate(k for k in keys if len(k.key) == prefix_len or k.key[prefix_len] in [".", ":"]):
            output.append({
                "key": strip_extension(k.key),
                "etag": convert.quote2string(k.etag),
                "expiry_date": Date(k.expiry_date),
                "last_modified": Date(k.last_modified)
            })
            if i >= limit:
                break
        return wrap(output)
714667
private JSONObject createResponseMeta(Map<String,String> metaInfo, Set<String> reqMetaInfo, Long ownerId, long requestTime)
            throws EventHandlerException, JSONException, ServiceException {
        JSONObject meta = new JsonObject();
        JSONObject headers = new JsonObject();

        for (String key : metaInfo.keySet()) {
            if (!Listener.AUTHENTICATED_USER_HEADER.equals(key)
                    && !Listener.METAINFO_HTTP_STATUS_CODE.equals(key)
                    && !Listener.METAINFO_ACCEPT.equals(key)
                    && !Listener.METAINFO_DOWNLOAD_FORMAT.equals(key)
                    && !Listener.METAINFO_MDW_REQUEST_ID.equals(key)
                    && !reqMetaInfo.contains(key)) {
                headers.put(key, metaInfo.get(key));
            }
        }

        // these always get populated if present
        if (metaInfo.get(Listener.METAINFO_REQUEST_ID) != null)
            headers.put(Listener.METAINFO_REQUEST_ID, metaInfo.get(Listener.METAINFO_REQUEST_ID));
        if (metaInfo.get(Listener.METAINFO_MDW_REQUEST_ID) != null && !metaInfo.get(Listener.METAINFO_MDW_REQUEST_ID).equals("0"))
            headers.put(Listener.METAINFO_MDW_REQUEST_ID, metaInfo.get(Listener.METAINFO_MDW_REQUEST_ID));
        if (metaInfo.get(Listener.METAINFO_CORRELATION_ID) != null)
            headers.put(Listener.METAINFO_CORRELATION_ID, metaInfo.get(Listener.METAINFO_CORRELATION_ID));
        if (metaInfo.get(Listener.METAINFO_CONTENT_TYPE) != null)
            headers.put(Listener.METAINFO_CONTENT_TYPE, metaInfo.get(Listener.METAINFO_CONTENT_TYPE));

        if (!headers.has(Listener.METAINFO_CONTENT_TYPE))
            headers.put(Listener.METAINFO_CONTENT_TYPE, "application/json");

        meta.put("headers", headers);

        createDocument(JSONObject.class.getName(), meta, OwnerType.LISTENER_RESPONSE_META, ownerId);


        ServiceLocator.getRequestServices().setElapsedTime(OwnerType.LISTENER_RESPONSE, ownerId,
                System.currentTimeMillis() - requestTime);

        return meta;
    }
637142
def _to_meta_data(pif_obj, dataset_hit, mdf_acl):
    """"""
    pif = pif_obj.as_dictionary()
    dataset = dataset_hit.as_dictionary()
    mdf = {}
    try:
        if pif.get("names"):
            mdf["title"] = pif["names"][0]
        else:
            mdf["title"] = "Citrine PIF " + str(pif["uid"])

        if pif.get("chemicalFormula"):
            mdf["composition"] = pif["chemicalFormula"]
        elif pif.get("composition"):
            mdf["composition"] = ''.join([comp["element"] for comp in pif["composition"] if comp["element"]])
        if not mdf["composition"]:
            mdf.pop("composition")

        mdf["acl"] = mdf_acl
        mdf["source_name"] = _construct_new_key(dataset["name"])

        if pif.get("contacts"):
            mdf["data_contact"] = []
            for contact in pif["contacts"]:
                data_c = {
                    "given_name": contact["name"]["given"],  #REQ
                    "family_name": contact["name"]["family"]  #REQ
                    }
                if contact.get("email"):
                    data_c["email"] = contact.get("email", "")
                if contact.get("orcid"):
                    data_c["orcid"] = contact.get("orcid", "")
                mdf["data_contact"].append(data_c)
            if not mdf["data_contact"]:
                mdf.pop("data_contact")
        
        mdf["data_contributor"] = [{}] 
        if "owner" in dataset:
            name = dataset["owner"].split()
            contributor = {
                "given_name": name[0],
                "family_name": name[1],
                "email": dataset["email"]
            }
            mdf["data_contributor"] = [contributor]

        mdf["links"] = {
            "landing_page": "https://citrination.com/datasets/{}".format(dataset["id"]),
            "publication": []
            }
        if pif.get("references"):
            mdf["author"] = []
            mdf["citation"] = []
            for ref in pif["references"]:
                if ref.get("doi"):
                    mdf["citation"].append(ref["doi"]) #TODO: Make actual citation
                    mdf["links"]["publication"].append(ref["doi"])
                if ref.get("authors"):
                    for author in ref["authors"]:
                        if author.get("given") and author.get("family"):
                            mdf["author"].append({
                                "given_name": author["given"],
                                "family_name": author["family"]
                                })
            # Remove fields if blank
            if not mdf["author"]:
                mdf.pop("author")
            if not mdf["citation"]:
                mdf.pop("citation")
        if not mdf["links"]["publication"]:
            mdf["links"].pop("publication")

        if pif.get("licenses", [{}])[0].get("url"):
            mdf["license"] = pif["licenses"][0]["url"]
        if pif.get("tags"):
            mdf["tags"] = pif["tags"]

    # If required MDF metadata is missing from PIF, abort
    except KeyError as e:
        print("Error: Required MDF metadata", str(e), "not found in PIF", pif["uid"])
        return None

    return mdf
526208
def ReadClientStats(self,
                      client_id,
                      min_timestamp,
                      max_timestamp,
                      cursor=None):
    """"""

    cursor.execute(
        """
        SELECT payload FROM client_stats
        WHERE client_id = %s
          AND timestamp BETWEEN FROM_UNIXTIME(%s) AND FROM_UNIXTIME(%s)
        ORDER BY timestamp ASC
        """, [
            db_utils.ClientIDToInt(client_id),
            mysql_utils.RDFDatetimeToTimestamp(min_timestamp),
            mysql_utils.RDFDatetimeToTimestamp(max_timestamp)
        ])
    return [
        rdf_client_stats.ClientStats.FromSerializedString(stats_bytes)
        for stats_bytes, in cursor.fetchall()
    ]
1150091
def aggregate_metrics(latencies_ms, duration_ms, num_bucket):
    """"""
    overall_op_count = 0
    op_counts = {operation : len(latency) for operation,
                 latency in latencies_ms.iteritems()}
    overall_op_count = sum([op_count for op_count in op_counts.itervalues()])

    print('[OVERALL], RunTime(ms), %f' % duration_ms)
    print('[OVERALL], Throughput(ops/sec), %f' % (float(overall_op_count) /
                                                duration_ms * 1000.0))

    for operation in op_counts.keys():
        operation_upper = operation.upper()
        print('[%s], Operations, %d' % (operation_upper, op_counts[operation]))
        print('[%s], AverageLatency(us), %f' % (
            operation_upper, numpy.average(latencies_ms[operation]) * 1000.0))
        print('[%s], LatencyVariance(us), %f' % (
            operation_upper, numpy.var(latencies_ms[operation]) * 1000.0))
        print('[%s], MinLatency(us), %f' % (
            operation_upper, min(latencies_ms[operation]) * 1000.0))
        print('[%s], MaxLatency(us), %f' % (
            operation_upper, max(latencies_ms[operation]) * 1000.0))
        print('[%s], 95thPercentileLatency(us), %f' % (
            operation_upper,
            numpy.percentile(latencies_ms[operation], 95.0) * 1000.0))
        print('[%s], 99thPercentileLatency(us), %f' % (
            operation_upper,
            numpy.percentile(latencies_ms[operation], 99.0) * 1000.0))
        print('[%s], 99.9thPercentileLatency(us), %f' % (
            operation_upper,
            numpy.percentile(latencies_ms[operation], 99.9) * 1000.0))
        print('[%s], Return=OK, %d' % (operation_upper, op_counts[operation]))
        latency_array = numpy.array(latencies_ms[operation])
        for j in range(num_bucket):
            print('[%s], %d, %d' % (
                operation_upper, j,
                ((j <= latency_array) & (latency_array < (j + 1))).sum()))
        print('[%s], >%d, %d' % (operation_upper, num_bucket,
                                 (num_bucket <= latency_array).sum()))
223003
def range_metadata(start, end, dst_folder, num_worker_threads=0, writers=[file_writer], geometry_check=None):
    

    assert isinstance(start, date)
    assert isinstance(end, date)

    delta = end - start

    dates = []

    for i in range(delta.days + 1):
        dates.append(start + timedelta(days=i))

    days = len(dates)

    total_counter = {
        'days': days,
        'products': 0,
        'saved_tiles': 0,
        'skipped_tiles': 0,
        'skipped_tiles_paths': []
    }

    def update_counter(counter):
        for key in iterkeys(total_counter):
            if key in counter:
                total_counter[key] += counter[key]

    for d in dates:
        logger.info('Getting metadata of {0}-{1}-{2}'.format(d.year, d.month, d.day))
        update_counter(daily_metadata(d.year, d.month, d.day, dst_folder, writers, geometry_check,
                                      num_worker_threads))

    return total_counter
721003
def mtime(self, key):
        """"""
        if key not in self.__dict:
            raise CacheKeyError(key)
        else:
            node = self.__dict[key]
            return node.mtime
529832
function avgCpuLoad (span, cb) {
  if (typeof span !== 'number' || span < 0) span = 100
  var start = os.cpus()

  setTimeout(function avgCpuLoadCompare () {
    var totals = os.cpus().reduce(function (totals, end, i) {
      var busy = end.times.user + end.times.sys - start[i].times.user - start[i].times.sys
      totals.total += busy + end.times.idle - start[i].times.idle
      totals.busy += busy
      return totals
    }, {busy: 0, total: 0})

    cb(totals.busy / totals.total)
  }, span)
}
474197
public void recordGetAllTime(long timeNS,
                                 int requested,
                                 int returned,
                                 long totalValueBytes,
                                 long totalKeyBytes) {
        recordTime(Tracked.GET_ALL,
                   timeNS,
                   requested - returned,
                   totalValueBytes,
                   totalKeyBytes,
                   requested);
    }
480605
def apply_meta(self, arr, funcs):
        
        if arr is None:
            logging.error("unable to convert data: emtpy section")
            return None

        tmp = np.copy(arr)
        for col in arr.dtype.names:
            for row in self._get_row_hdrs():
                meta = self._get_meta(row, col)
                for mk, mv in sorted(meta.iteritems()):
                    if mk in funcs.iterkeys():
                        tmp[col][self.irt[row]] = \
                            funcs[mk](tmp[col][self.irt[row]], mv)
        return tmp
1207261
function getMetadata (dir) {
  var json = path.join(dir, 'meta.json')
  var js = path.join(dir, 'meta.js')
  var opts = {}

  if (exists(json)) {
    opts = metadata.sync(json)
  } else if (exists(js)) {
    var req = require(path.resolve(js))
    if (req !== Object(req)) {
      throw new Error('meta.js needs to expose an object')
    }
    opts = req
  }

  return opts
}