Export collected metrics to CopperEgg
770744
@Override
    public void write(Iterable<QueryResult> results) {

        List<QueryResult> jvm_os_counters = new ArrayList<QueryResult>();
        List<QueryResult> jvm_gc_counters = new ArrayList<QueryResult>();
        List<QueryResult> jvm_runtime_counters = new ArrayList<QueryResult>();
        List<QueryResult> jvm_class_counters = new ArrayList<QueryResult>();
        List<QueryResult> jvm_thread_counters = new ArrayList<QueryResult>();
        List<QueryResult> heap_counters = new ArrayList<QueryResult>();
        List<QueryResult> nonheap_counters = new ArrayList<QueryResult>();
        List<QueryResult> tomcat_thread_pool_counters = new ArrayList<QueryResult>();
        List<QueryResult> tomcat_grp_counters = new ArrayList<QueryResult>();
        List<QueryResult> tomcat_manager_counters = new ArrayList<QueryResult>();
        List<QueryResult> tomcat_servlet_counters = new ArrayList<QueryResult>();
        List<QueryResult> tomcat_db_counters = new ArrayList<QueryResult>();
        List<QueryResult> jmxtrans_counters = new ArrayList<QueryResult>();
        List<QueryResult> app_counters = new ArrayList<QueryResult>();
        List<QueryResult> app_sales_counters = new ArrayList<QueryResult>();

        long epochInMillis = 0;
        String myname =  null;
        Object myval = null;
        long thisPID = 0;
        String tmp = null;
        String pidHost = null;

        String delims = "[.]";
        for (QueryResult result : results) {
            epochInMillis = result.getEpochInMillis();
            myname =  result.getName();
            myval = result.getValue();
            String valstr = myval.toString();
            thisPID = getPID();
            tmp = String.valueOf(thisPID);
            pidHost = source + "." + tmp;

            String[] parts = myname.split(delims);
            if( parts.length > 0 ) {
                String p1 = parts[0];
                if( (jmxtrans_metric_groupID != null) && (p1.equals("jmxtrans")) )  {
                    QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                    jmxtrans_counters.add(new_result);
                } else if( p1.equals("jvm") ) {
                    if( parts[1].equals("os")) {
                        if (parts[2].equals("OpenFileDescriptorCount")) {
                            QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                            jvm_os_counters.add(new_result);
                        } else if (parts[2].equals("CommittedVirtualMemorySize")){
                            float fval = Float.parseFloat(valstr);
                            try {
                                fval = fval/(1024.0f*1024.0f);  
                            } catch (Exception e) {
                                exceptionCounter.incrementAndGet();
                                logger.info("Exception doing Float: ", e);
                            }
                            QueryResult new_result = new QueryResult(myname, pidHost, fval, epochInMillis);
                            jvm_os_counters.add(new_result);
                        } else if (parts[2].equals("ProcessCpuTime")) { 
                            float fval = Float.parseFloat(valstr);
                            try {
                                fval = fval/(1000.0f*1000.0f*1000.0f);  
                            } catch (Exception e) {
                                exceptionCounter.incrementAndGet();
                                logger.warn("Exception doing Float: ", e);
                            }
                            QueryResult new_result = new QueryResult(myname, pidHost, fval, epochInMillis);
                            jvm_os_counters.add(new_result);
                        }
                    } else if( (parts[1].equals("runtime")) && (parts[2].equals("Uptime")) ) { 
                        float fval = Float.parseFloat(valstr);
                        try {
                            fval = fval/(1000.0f*60.0f);  
                        } catch (Exception e) {
                            exceptionCounter.incrementAndGet();
                            logger.warn("Exception doing Float: ", e);
                        }
                        QueryResult new_result = new QueryResult(myname, pidHost, fval, epochInMillis);
                        jvm_runtime_counters.add(new_result); 
                    } else if( (parts[1].equals("loadedClasses")) && (parts[2].equals("LoadedClassCount")) ) {
                        // jvm.loadedClasses.LoadedClassCount 5099 1374549969
                        QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                        jvm_class_counters.add(new_result); 
                    } else if( (parts[1].equals("thread")) && (parts[2].equals("ThreadCount")) ){
                        // jvm.thread.ThreadCount 13 1374549940
                        QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                        jvm_thread_counters.add(new_result);
                    } else if( (parts[1].equals("gc")) &&
                        ( (parts[2].equals("Copy")) ||  (parts[2].equals("MarkSweepCompact")) ) &&
                        ( (parts[3].equals("CollectionCount")) ||  (parts[3].equals("CollectionTime")) ) ) {
                        // jvm.gc.Copy.CollectionCount 68 1374549940
                        QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                        jvm_gc_counters.add(new_result);
                    } else if( parts[1].equals("memorypool") ){ 
                        if( ( (parts[2].equals("Perm_Gen")) || (parts[2].equals("Code_Cache")) ) && 
                            ( (parts[4].equals("committed")) || (parts[4].equals("used"))  ) ) {     
                            myname = "jvmNonHeapMemoryUsage";
                            String fullID = pidHost + "." + parts[2] + "." + parts[4];
                            float fval = Float.parseFloat(valstr);
                            try {
                                fval = fval/(1024.0f*1024.0f);
                            } catch (Exception e) {
                                exceptionCounter.incrementAndGet();
                                logger.warn("Exception doing Float: ", e);
                            }
                            QueryResult new_result = new QueryResult(myname, fullID, fval, epochInMillis);
                            nonheap_counters.add(new_result);
                        } else if( ( (parts[2].equals("Eden_Space")) || 
                                     (parts[2].equals("Survivor_Space")) || 
                                     (parts[2].equals("Tenured_Gen")) 
                                    ) && (  (parts[4].equals("committed")) ||  (parts[4].equals("used"))  ) ) {    
                            myname = "jvmHeapMemoryUsage";
                            String fullID = pidHost + "." + parts[2] + "." + parts[4];
                            float fval = Float.parseFloat(valstr);
                            try {
                                fval = fval/(1024.0f*1024.0f);
                            } catch (Exception e) {
                                exceptionCounter.incrementAndGet();
                                logger.warn("Exception doingFloat: ", e);
                            }
                            QueryResult new_result = new QueryResult(myname, fullID, fval, epochInMillis);
                            heap_counters.add(new_result);
                        }
                    } 
                } else if( p1.equals("tomcat") ) {
                    if( (parts[1].equals("thread-pool")) &&  
                         ( (parts[3].equals("currentThreadsBusy")) || (parts[3].equals("currentThreadCount")) ) ){
                        // tomcat.thread_pool.http-bio-8080.currentThreadCount 0 1374549955
                        String connector = parts[2];
                        myname = parts[0] + "." + parts[1] + "." + parts[3];
                        String fullID = pidHost + "." + connector;
                        QueryResult new_result = new QueryResult(myname, fullID, myval, epochInMillis);
                        tomcat_thread_pool_counters.add(new_result);
                   } else if( parts[1].equals("global-request-processor") ) {
                        // tomcat.global-request-processor.http-bio-8080.bytesSent
                        String connector = parts[2];
                        myname = parts[0] + "." + parts[1] + "." + parts[3];
                        String fullID = pidHost + "." + connector;
                        if( parts[3].equals("processingTime")) {
                            float fval = Float.parseFloat(valstr);
                            try {
                                fval = fval/(1024.0f);
                            } catch (Exception e) {
                                exceptionCounter.incrementAndGet();
                                logger.warn("Exception doingFloat: ", e);
                            }
                            QueryResult new_result = new QueryResult(myname, fullID, fval, epochInMillis);
                            tomcat_grp_counters.add(new_result);
                        } else {
                            QueryResult new_result = new QueryResult(myname, fullID, myval, epochInMillis);
                            tomcat_grp_counters.add(new_result);
                        }
                    } else if( (parts[1].equals("manager"))  && (parts[4].equals("activeSessions")) ){
                        //  tomcat.manager.localhost._docs.activeSessions 0 1374549955
                        String myhost = parts[2];
                        String mycontext = parts[3];
                        myname = parts[0] + "." + parts[1] + "." + parts[4];
                        String fullID = pidHost + "." + myhost + "." + mycontext;
                        QueryResult new_result = new QueryResult(myname, fullID, myval, epochInMillis);
                        tomcat_manager_counters.add(new_result);
                    } else if( (parts[1].equals("servlet")) && 
                          ( (parts[4].equals("processingTime")) || 
                            (parts[4].equals("errorCount")) || 
                            (parts[4].equals("requestCount")) ) ){
                        // tomcat.servlet.__localhost_cocktail-app-1_0_9-SNAPSHOT.spring-mvc.processingTime
                        String myWebmodule = parts[2];
                        String myServletname = parts[3];
                        myname = parts[0] + "." + parts[1] + "." + parts[4];
                        String fullID = pidHost + "." + myWebmodule + "." + myServletname;
                        QueryResult new_result = new QueryResult(myname, fullID, myval, epochInMillis);
                        tomcat_servlet_counters.add(new_result);
                    } else if( (tomcat_db_groupID != null) && (parts[1].equals("data-source")) ) {                   
                        String myhost = parts[2];
                        String mycontext = parts[3];
                        String mydbname = parts[4];
                        myname = parts[0] + "." + parts[1] + "." + parts[5];
                        String fullID = pidHost + "." + myhost + "." + mycontext + "." + mydbname;
                        QueryResult new_result = new QueryResult(myname, fullID, myval, epochInMillis);
                        tomcat_db_counters.add(new_result);
                    }
                } else if( (app_groupID != null) && (p1.equals("cocktail")) ) {
                    if( !(parts[1].equals("CreatedCocktailCount")) &&  !(parts[1].equals("UpdatedCocktailCount")) ) {
                        QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                        app_counters.add(new_result);
                    }
                } else if( ( (app_sales_groupID != null) && (p1.equals("sales")) ) &&
                      ( (parts[1].equals("ordersCounter")) || 
                        (parts[1].equals("itemsCounter")) || 
                        (parts[1].equals("revenueInCentsCounter")) ) ){
                        QueryResult new_result = new QueryResult(myname, pidHost, myval, epochInMillis);
                        app_sales_counters.add(new_result);
                }
            } else {
                logger.warn("parts return NULL!!!");
            }
        }
        if(!jvm_os_counters.isEmpty()) {
            sort_n_send(jvm_os_groupID, jvm_os_counters);
        }
        if(!jvm_gc_counters.isEmpty()) {
            sort_n_send(jvm_gc_groupID, jvm_gc_counters);
        }
        if(!jvm_runtime_counters.isEmpty()) {
            sort_n_send(jvm_runtime_groupID, jvm_runtime_counters);
        }
        if(!jvm_class_counters.isEmpty()) {
            sort_n_send(jvm_class_groupID, jvm_class_counters);
        }
        if(!jvm_thread_counters.isEmpty()) {
            sort_n_send(jvm_thread_groupID, jvm_thread_counters);
        }
        if(!heap_counters.isEmpty()) {
            sort_n_send(heap_metric_groupID, heap_counters);
        }
        if(!nonheap_counters.isEmpty()) {
            sort_n_send(nonheap_metric_groupID, nonheap_counters);
        }
        if(!tomcat_thread_pool_counters.isEmpty()) {
            sort_n_send(tomcat_thread_pool_groupID, tomcat_thread_pool_counters);
        }
        if(!tomcat_grp_counters.isEmpty()) {
            sort_n_send(tomcat_grp_groupID,tomcat_grp_counters);
        }
        if(!tomcat_servlet_counters.isEmpty()) {
            sort_n_send(tomcat_servlet_groupID, tomcat_servlet_counters);
        }
        if(!tomcat_manager_counters.isEmpty()) {
            sort_n_send(tomcat_manager_groupID, tomcat_manager_counters);
        }
        if(!tomcat_db_counters.isEmpty()) {
            sort_n_send(tomcat_db_groupID, tomcat_db_counters);
        }
        if(!jmxtrans_counters.isEmpty()) {
            sort_n_send(jmxtrans_metric_groupID, jmxtrans_counters);
        }
        if(!app_counters.isEmpty()) {
            sort_n_send(app_groupID, app_counters);
        }
        if(!app_sales_counters.isEmpty()) {
            sort_n_send(app_sales_groupID, app_sales_counters);
        }
    }
338734
def export(self, path, session):
    """"""
    def variables_saver(variables_path):
      if self._saver:
        self._saver.save(
            session, variables_path,
            write_meta_graph=False,
            write_state=False)

    self._spec._export(path, variables_saver)
580905
def export(id, local=False, scrub_pii=False):
    """"""

    print("Preparing to export the data...")

    if local:
        db_uri = db.db_url
    else:
        db_uri = HerokuApp(id).db_uri

    # Create the data package if it doesn't already exist.
    subdata_path = os.path.join("data", id, "data")
    try:
        os.makedirs(subdata_path)

    except OSError as e:
        if e.errno != errno.EEXIST or not os.path.isdir(subdata_path):
            raise

    # Copy in the data.
    copy_db_to_csv(db_uri, subdata_path, scrub_pii=scrub_pii)

    # Copy the experiment code into a code/ subdirectory.
    try:
        shutil.copyfile(
            os.path.join("snapshots", id + "-code.zip"),
            os.path.join("data", id, id + "-code.zip"),
        )

    except Exception:
        pass

    # Copy in the DATA readme.
    # open(os.path.join(id, "README.txt"), "a").close()

    # Save the experiment id.
    with open(os.path.join("data", id, "experiment_id.md"), "a+") as file:
        file.write(id)

    # Zip data
    src = os.path.join("data", id)
    dst = os.path.join("data", id + "-data.zip")
    archive_data(id, src, dst)

    cwd = os.getcwd()
    data_filename = "{}-data.zip".format(id)
    path_to_data = os.path.join(cwd, "data", data_filename)

    # Backup data on S3 unless run locally
    if not local:
        bucket = user_s3_bucket()
        bucket.upload_file(path_to_data, data_filename)
        url = _generate_s3_url(bucket, data_filename)

        # Register experiment UUID with dallinger
        register(id, url)

    return path_to_data
1309326
def _create_extractors(cls, metrics):
        
        metrics_dict = {}
        # group entries by `category` attribute (default is "standard")
        for metric, config in six.iteritems(metrics):
            category = config.get('category', StdBenchmark.DEFAULT_CATEGORY)
            metrics_dict.setdefault(category, {})[metric] = config
        # create one StdExtractor instance per category,
        # passing associated metrics
        return dict(
            (category, StdExtractor(metrics))
            for category, metrics in six.iteritems(metrics_dict)
        )
496035
def _export_profiles(self, profile_name, user_pageviews, user_downloads,
                         ip_user=False):
        """"""
        views_min = self.config.get('user_views_min')
        views_max = self.config.get('user_views_max')
        ip_user_id = 500000000000
        add_user_id = 100000000000
        stat_records = 0
        with self.storage.get_user_profiles(profile_name) as store:
            store.clear()
            for user in user_pageviews:
                # Only users with unique pageviews.
                unique_views = len(set(user_pageviews[user]))
                if views_max > unique_views >= views_min:
                    nodes, weight = self._calculate_user_record_weights(
                            record_list=user_pageviews[user],
                            download_list=user_downloads.get(user))
                    if ip_user:
                        store.add_user(ip_user_id, nodes, weight)
                        ip_user_id += 1
                    else:
                        user = str(add_user_id + int(user))
                        store.add_user(user, nodes, weight)
                    self.stat_long['User_num_records'].append(len(nodes))
                    stat_records += len(nodes)

                elif unique_views >= views_min:
                    # TODO: Add stat for to many views.
                    print("Drop user {} with {} views".format(user,
                                                              unique_views))
        self.stat['user_profiles'] = len(self.stat_long.get(
                                                        'User_num_records'))
        self.stat['user_profiles_records'] = stat_records

        print("Stats: {}".format(self.stat))
694534
def _merge_metrics(samples, out_dir):
    
    logger.info("summarize metrics")
    out_dir = utils.safe_makedir(os.path.join(out_dir, "report", "metrics"))
    sample_metrics = collections.defaultdict(dict)
    for s in samples:
        s = _add_disambiguate(s)
        m = tz.get_in(['summary', 'metrics'], s)
        if isinstance(m, six.string_types):
            m = json.loads(m)
        if m:
            for me in m.keys():
                if isinstance(m[me], list) or isinstance(m[me], dict) or isinstance(m[me], tuple):
                    m.pop(me, None)
            sample_metrics[dd.get_sample_name(s)].update(m)
    out = []
    for sample_name, m in sample_metrics.items():
        sample_file = os.path.join(out_dir, "%s_bcbio.txt" % sample_name)
        with file_transaction(samples[0], sample_file) as tx_out_file:
            dt = pd.DataFrame(m, index=['1'])
            dt.columns = [k.replace(" ", "_").replace("(", "").replace(")", "") for k in dt.columns]
            dt['sample'] = sample_name
            dt['rRNA_rate'] = m.get('rRNA_rate', "NA")
            dt['RiP_pct'] = "%.3f" % (int(m.get("RiP", 0)) / float(m.get("Total_reads", 1)) * 100)
            dt = _fix_duplicated_rate(dt)
            dt.transpose().to_csv(tx_out_file, sep="\t", header=False)
        out.append(sample_file)
    return out
93528
def export()
      super
      f = new_output_file('obo_nomenclature.obo') 
     
      # header 
      f.puts 'format-version: 1.2'
      f.puts "date: #{@time}"
      f.puts 'saved-by: someone'
      f.puts 'auto-generated-by: Taxonifi'
      f.puts 'synonymtypedef: COMMONNAME "common name"'
      f.puts 'synonymtypedef: MISSPELLING "misspelling" EXACT'
      f.puts 'synonymtypedef: TAXONNAMEUSAGE "name with (author year)" NARROW'
      f.puts "default-namespace: #{@namespace}"
      f.puts "ontology: FIX-ME-taxonifi-ontology\n\n" 

      # terms
      @name_collection.collection.each do |n|
        f.puts '[Term]'
        f.puts "id: #{id_string(n)}"
        f.puts "name: #{n.name}"
        f.puts "is_a: #{id_string(n.parent)} ! #{n.parent.name}" if n.parent
        f.puts "property_value: has_rank #{rank_string(n)}"
        f.puts
      end

      # typedefs
      f.puts "[Typedef]"
      f.puts "id: has_rank"
      f.puts "name: has taxonomic rank"
      f.puts "is_metadata_tag: true"

      true
    end
554161
def SetupPrometheusExportsFromConfig():
    """"""
    port = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_PORT', None)
    port_range = getattr(
        settings, 'PROMETHEUS_METRICS_EXPORT_PORT_RANGE', None)
    addr = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_ADDRESS', '')
    if port_range:
        SetupPrometheusEndpointOnPortRange(port_range, addr)
    elif port:
        SetupPrometheusEndpointOnPort(port, addr)
884698
def _collect_stats(
        self,
        key,
        db,
        instance_tags,
        relations,
        custom_metrics,
        collect_function_metrics,
        collect_count_metrics,
        collect_activity_metrics,
        collect_database_size_metrics,
        collect_default_db,
        interface_error,
        programming_error,
    ):
        

        db_instance_metrics = self._get_instance_metrics(key, db, collect_database_size_metrics, collect_default_db)
        bgw_instance_metrics = self._get_bgw_metrics(key, db)
        archiver_instance_metrics = self._get_archiver_metrics(key, db)

        metric_scope = [self.CONNECTION_METRICS, self.LOCK_METRICS]

        if collect_function_metrics:
            metric_scope.append(self.FUNCTION_METRICS)
        if collect_count_metrics:
            metric_scope.append(self.COUNT_METRICS)

        # Do we need relation-specific metrics?
        relations_config = {}
        if relations:
            metric_scope += [self.REL_METRICS, self.IDX_METRICS, self.SIZE_METRICS, self.STATIO_METRICS]
            relations_config = self._build_relations_config(relations)

        replication_metrics = self._get_replication_metrics(key, db)
        if replication_metrics is not None:
            # FIXME: constants shouldn't be modified
            self.REPLICATION_METRICS['metrics'] = replication_metrics
            metric_scope.append(self.REPLICATION_METRICS)

        try:
            cursor = db.cursor()
            results_len = self._query_scope(
                cursor, db_instance_metrics, key, db, instance_tags, False, programming_error, relations_config
            )
            if results_len is not None:
                self.gauge(
                    "postgresql.db.count", results_len, tags=[t for t in instance_tags if not t.startswith("db:")]
                )

            self._query_scope(
                cursor, bgw_instance_metrics, key, db, instance_tags, False, programming_error, relations_config
            )
            self._query_scope(
                cursor, archiver_instance_metrics, key, db, instance_tags, False, programming_error, relations_config
            )

            if collect_activity_metrics:
                activity_metrics = self._get_activity_metrics(key, db)
                self._query_scope(
                    cursor, activity_metrics, key, db, instance_tags, False, programming_error, relations_config
                )

            for scope in list(metric_scope) + custom_metrics:
                self._query_scope(
                    cursor, scope, key, db, instance_tags, scope in custom_metrics, programming_error, relations_config
                )

            cursor.close()
        except (interface_error, socket.error) as e:
            self.log.error("Connection error: %s" % str(e))
            raise ShouldRestartException
1358278
def _export_signatures(meta_graph):
  """"""
  named_signatures = tf_v1.get_collection(_SIGNATURE_COLLECTION)
  if not named_signatures:
    raise ValueError("No signatures present. Please call hub.add_signature(...)"
                     "at least once in the module_fn.")
  for key, signature in named_signatures:
    meta_graph.signature_def[key].CopyFrom(signature)
459224
def export(self):
        
        data = {}
        data["name"] = self.name
        data["contributions"] = self.contributions
        data["avatar"] = self.avatar
        data["followers"] = self.followers
        data["join"] = self.join
        data["organizations"] = self.organizations
        data["repositories"] = self.numberOfRepos
        data["bio"] = self.bio
        data["private"] = self.private
        data["public"] = self.public
        data["location"] = self.location
        return data
447625
def export(self):
        
        with LogTask('Exporting disk {} to {}'.format(self.name, self.dst)):
            with utils.RollbackContext() as rollback:
                rollback.prependDefer(
                    shutil.rmtree, self.dst, ignore_errors=True
                )
                self.copy()
                if not self.disk['format'] == 'iso':
                    self.sparse()
                self.calc_sha('sha1')
                self.update_lago_metadata()
                self.write_lago_metadata()
                self.compress()
                rollback.clear()
59703
def export_gpx_file(self):
        
        gpx = create_elem('gpx', GPX_ELEM_ATTRIB)
        if not self.metadata.bounds:
            self.metadata.bounds = self[:]
        gpx.append(self.metadata.togpx())
        for place in self:
            gpx.append(place.togpx())

        return etree.ElementTree(gpx)
937532
def bulk_write(self, metrics):
        
        actions = []
        index = self.get_index()
        for metric in metrics:
            actions.append({'index': {'_index': index, '_type': self.doc_type}})
            actions.append(metric)
        try:
            self.client.bulk(actions)
        except TransportError as exc:
            logger.warning('bulk_write metrics %r failure %r', metrics, exc)
1273232
def dump_stats(self, fdump, close=True):
        
        if self.tracker:
            self.tracker.stop_periodic_snapshots()

        if isinstance(fdump, type('')):
            fdump = open(fdump, 'wb')
        pickle.dump(self.index, fdump, protocol=pickle.HIGHEST_PROTOCOL)
        pickle.dump(self.snapshots, fdump, protocol=pickle.HIGHEST_PROTOCOL)
        if close:
            fdump.close()
1140307
def output_metric(gandi, metrics, key, justify=10):
    
    for metric in metrics:
        key_name = metric[key].pop()
        values = [point.get('value', 0) for point in metric['points']]
        graph = sparks(values) if max(values) else ''
        # need to encode in utf-8 to work in python2.X
        if sys.version_info < (2, 8):
            graph = graph.encode('utf-8')
        output_line(gandi, key_name, graph, justify)
608625
def generate_exports():
	
	env = []
	for name in list_installed_genomes():
	    try:
	        g = Genome(name)
	        env_name = re.sub(r'[^\w]+', "_", name).upper()
	        env.append("export {}={}".format(env_name, g.filename))
	    except:
	        pass
	return env