Parse MIP qc metrics file.
    Args:
        metrics (dict): raw YAML input from MIP qc metrics file

    Returns:
        dict: parsed data
766090
def parse_qcmetrics(metrics: dict) -> dict:
    
    data = {
        'versions': {
            'freebayes': metrics['program']['freebayes']['version'],
            'gatk': metrics['program']['gatk']['version'],
            'manta': metrics['program'].get('manta', {}).get('version'),
            'bcftools': metrics['program']['bcftools']['version'],
            'vep': metrics['program']['varianteffectpredictor']['version'],
        },
        'samples': [],
    }

    plink_samples = {}
    plink_sexcheck = metrics['program'].get('plink_sexcheck', {}).get('sample_sexcheck')
    if isinstance(plink_sexcheck, str):
        sample_id, sex_number = plink_sexcheck.strip().split(':', 1)
        plink_samples[sample_id] = PED_SEX_MAP.get(int(sex_number))
    elif isinstance(plink_sexcheck, list):
        for sample_raw in plink_sexcheck:
            sample_id, sex_number = sample_raw.split(':', 1)
            plink_samples[sample_id] = PED_SEX_MAP.get(int(sex_number))

    for sample_id, sample_metrics in metrics['sample'].items():

        ## Bam stats metrics
        bam_stats = [values['bamstats'] for key, values in sample_metrics.items()
                     if key[:-1].endswith('.lane')]
        total_reads = sum(int(bam_stat['raw_total_sequences']) for bam_stat in bam_stats)
        total_mapped = sum(int(bam_stat['reads_mapped']) for bam_stat in bam_stats)

        ## Picard metrics
        main_key = [key for key in sample_metrics.keys() if '_lanes_' in key][0]

        hs_metrics = sample_metrics[main_key]['collecthsmetrics']['header']['data']
        multiple_inst_metrics = sample_metrics[main_key]['collectmultiplemetricsinsertsize']['header']['data']
        multiple_metrics = sample_metrics[main_key]['collectmultiplemetrics']['header']['pair']

        sample_data = {
            'at_dropout': hs_metrics['AT_DROPOUT'],
            'completeness_target': {
                10: hs_metrics['PCT_TARGET_BASES_10X'],
                20: hs_metrics['PCT_TARGET_BASES_20X'],
                50: hs_metrics['PCT_TARGET_BASES_50X'],
                100: hs_metrics['PCT_TARGET_BASES_100X'],
            },
            'duplicates': float(sample_metrics[main_key]['markduplicates']['fraction_duplicates']),
            'gc_dropout': hs_metrics['GC_DROPOUT'],
            'id': sample_id,
            'median_insert_size':  multiple_inst_metrics['MEDIAN_INSERT_SIZE'],
            'mapped': total_mapped / total_reads,
            'plink_sex': plink_samples.get(sample_id),
            'predicted_sex': sample_metrics[main_key]['chanjo_sexcheck']['gender'],
            'reads': total_reads,
            'insert_size_standard_deviation': float(multiple_inst_metrics['STANDARD_DEVIATION']),
            'strand_balance': float(multiple_metrics['STRAND_BALANCE']),
            'target_coverage': float(hs_metrics['MEAN_TARGET_COVERAGE']),
        }
        data['samples'].append(sample_data)
    return data
266599
def _match_metric(self, metric):
        
        if len(self._compiled_filters) == 0:
            return True
        for (collector, filter_regex) in self._compiled_filters:
            if collector != metric.getCollectorPath():
                continue
            if filter_regex.match(metric.getMetricPath()):
                return True
        return False
393029
def export_metrics(self, metrics):
        
        metric_protos = []
        for metric in metrics:
            metric_protos.append(_get_metric_proto(metric))

        self._rpc_handler.send(
            metrics_service_pb2.ExportMetricsServiceRequest(
                metrics=metric_protos))
815213
def parse_reports(self):
    

    # Set up vars
    self.picard_pcrmetrics_data = dict()
    self.picard_pcrmetrics_samplestats = dict()

    # Go through logs and find Metrics
    for f in self.find_log_files('picard/pcr_metrics', filehandles=True):
        s_name = None
        for l in f['f']:
            # New log starting
            if 'TargetedPcrMetrics' in l and 'INPUT' in l:
                s_name = None
                # Pull sample name from input
                fn_search = re.search(r"INPUT(?:=|\s+)(\[?[^\s]+\]?)", l, flags=re.IGNORECASE)
                if fn_search:
                    s_name = os.path.basename(fn_search.group(1).strip('[]'))
                    s_name = self.clean_s_name(s_name, f['root'])

            if s_name is not None:
                if 'TargetedPcrMetrics' in l and '## METRICS CLASS' in l:
                    keys = f['f'].readline().strip("\n").split("\t")
                    vals = f['f'].readline().strip("\n").split("\t")
                    if len(vals) == len(keys):
                        if s_name in self.picard_pcrmetrics_data:
                            log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
                        self.add_data_source(f, s_name, section='TargetedPcrMetrics')
                        self.picard_pcrmetrics_data[s_name] = dict()
                        for i, k in enumerate(keys):
                            try:
                                # Multiply percentages by 100
                                if k.startswith('PCT_'):
                                    vals[i] = float(vals[i]) * 100.0
                                self.picard_pcrmetrics_data[s_name][k] = float(vals[i])
                            except ValueError:
                                self.picard_pcrmetrics_data[s_name][k] = vals[i]

    # Filter to strip out ignored sample names
    self.picard_pcrmetrics_data = self.ignore_samples(self.picard_pcrmetrics_data)

    if len(self.picard_pcrmetrics_data) > 0:

        # Write parsed data to a file
        self.write_data_file(self.picard_pcrmetrics_data, 'multiqc_picard_pcrmetrics')

        # Add to general stats table
        self.general_stats_headers['PCT_AMPLIFIED_BASES'] = {
            'title': '% Amplified Bases',
            'description': 'The fraction of aligned bases that mapped to or near an amplicon.',
            'min': 0,
            'max': 100,
            'suffix': '%',
            'scale': 'BrBG'
        }
        self.general_stats_headers['MEDIAN_TARGET_COVERAGE'] = {
            'title': 'Median Target Coverage',
            'description': 'The median coverage of reads that mapped to target regions of an experiment.',
            'min': 0,
            'suffix': 'X',
            'scale': 'GnBu',
        }

        for s_name in self.picard_pcrmetrics_data:
            if s_name not in self.general_stats_data:
                self.general_stats_data[s_name] = dict()
            self.general_stats_data[s_name].update( self.picard_pcrmetrics_data[s_name] )

        # Bar plot of ignored bases
        keys = OrderedDict()
        keys['ON_AMPLICON_BASES'] = {'name': 'On-amplicon bases'}
        keys['NEAR_AMPLICON_BASES'] = {'name': 'Near-amplicon bases'}
        keys['OFF_AMPLICON_BASES'] = {'name': 'Off-amplicon bases', 'color': '#f28f43'}

        # Config for the plot
        pconfig = {
            'id': 'picard_pcr_metrics_bases',
            'title': 'Picard: PCR Amplicon Bases',
            'ylab': '# Bases',
            'cpswitch_counts_label': '# Bases',
            'hide_zero_cats': False
        }

        self.add_section (
            name = 'PCR Amplicon Bases',
            anchor = 'picard-pcrmetrics-bases',
            description = 'Metrics about reads obtained from targeted PCR experiments.',
            helptext = '''
            This plot shows the number of bases aligned on or near to amplified regions of the genome.

            * `ON_AMPLICON_BASES`: The number of `PF_BASES_ALIGNED` that mapped to an amplified region of the genome.
            * `NEAR_AMPLICON_BASES`: The number of `PF_BASES_ALIGNED` that mapped to within a fixed interval of an amplified region, but not on a baited region.
            * `OFF_AMPLICON_BASES`: The number of `PF_BASES_ALIGNED` that mapped neither on or near an amplicon.

            For more information see the [Picard documentation](https://broadinstitute.github.io/picard/picard-metric-definitions.html#TargetedPcrMetrics).''',
            plot = bargraph.plot(self.picard_pcrmetrics_data, keys, pconfig)
        )


    # Return the number of detected samples to the parent module
    return len(self.picard_pcrmetrics_data)
806325
def metric(self, measurement_name, values, tags=None, timestamp=None):
        
        if not measurement_name or values in (None, {}):
            # Don't try to send empty data
            return

        tags = tags or {}

        # Do a shallow merge of the metric tags and global tags
        all_tags = dict(self.tags, **tags)

        # Create a metric line from the input and then send it to socket
        line = Line(measurement_name, values, all_tags, timestamp)
        self.send(line.to_line_protocol())
1110338
def _format_disk_metrics(self, metrics):
        """"""
        for name, raw_val in metrics.iteritems():
            if raw_val:
                match = DISK_STATS_RE.search(raw_val)
                if match is None or len(match.groups()) != 2:
                    self.log.warning('Can\'t parse value %s for disk metric %s. Dropping it.' % (raw_val, name))
                    metrics[name] = None
                val, unit = match.groups()
                # by default some are uppercased others lowercased. That's error prone.
                unit = unit.lower()
                try:
                    val = int(float(val) * UNIT_MAP[unit])
                    metrics[name] = val
                except KeyError:
                    self.log.error('Unrecognized unit %s for disk metric %s. Dropping it.' % (unit, name))
                    metrics[name] = None
        return metrics
443262
def get_custom_metrics():
    
    _rse = mx.metric.create(rse)
    _rae = mx.metric.create(rae)
    _corr = mx.metric.create(corr)
    return mx.metric.create([_rae, _rse, _corr])
977370
async def get_metrics(self, *tags):
        
        log.debug("Retrieving metrics for %s",
                  ', '.join(tags) if tags else "all units")

        metrics_facade = client.MetricsDebugFacade.from_connection(
            self.connection())

        entities = [client.Entity(tag) for tag in tags]
        metrics_result = await metrics_facade.GetMetrics(entities)

        metrics = collections.defaultdict(list)

        for entity_metrics in metrics_result.results:
            error = entity_metrics.error
            if error:
                if "is not a valid tag" in error:
                    raise ValueError(error.message)
                else:
                    raise Exception(error.message)

            for metric in entity_metrics.metrics:
                metrics[metric.unit].append(vars(metric))

        return metrics
1238050
def process_build_metrics(context, build_processors):
    """"""
    build_metrics = OrderedDict()

    # reset all processors
    for p in build_processors:
        p.reset()

    # collect metrics from all processors
    for p in build_processors:
        build_metrics.update(p.build_metrics)

    return build_metrics
862411
def metadata(check):
    
    metric_sources = get_metric_sources()

    if check:
        if check not in metric_sources:
            abort('Metadata file `{}` does not exist.'.format(get_metadata_file(check)))
        metric_sources = [check]
    else:
        metric_sources = sorted(metric_sources)

    errors = False

    for current_check in metric_sources:
        if current_check.startswith('datadog_checks_'):
            continue

        # get any manifest info needed for validation
        manifest = load_manifest(current_check)
        try:
            metric_prefix = manifest['metric_prefix'].rstrip('.')
        except KeyError:
            metric_prefix = None

        metadata_file = get_metadata_file(current_check)

        # To make logging less verbose, common errors are counted for current check
        metric_prefix_count = defaultdict(int)
        empty_count = defaultdict(int)
        empty_warning_count = defaultdict(int)
        duplicate_set = set()
        metric_prefix_error_shown = False

        # Python 2 csv module does not support unicode
        with open(metadata_file, 'rb' if PY2 else 'r', encoding=None if PY2 else 'utf-8') as f:
            reader = csv.DictReader(f, delimiter=',')

            # Read header
            if PY2:
                reader._fieldnames = [key.decode('utf-8') for key in reader.fieldnames]
            else:
                reader._fieldnames = reader.fieldnames

            for row in reader:

                # Number of rows is correct. Since metric is first in the list, should be safe to access
                if len(row) != len(ALL_HEADERS):
                    errors = True
                    echo_failure('{}: {} Has the wrong amount of columns'.format(current_check, row['metric_name']))
                    continue

                if PY2:
                    for key, value in iteritems(row):
                        if value is not None:
                            row[key] = value.decode('utf-8')

                # all headers exist, no invalid headers
                all_keys = set(row)
                if all_keys != ALL_HEADERS:
                    invalid_headers = all_keys.difference(ALL_HEADERS)
                    if invalid_headers:
                        errors = True
                        echo_failure('{}: Invalid column {}'.format(current_check, invalid_headers))

                    missing_headers = ALL_HEADERS.difference(all_keys)
                    if missing_headers:
                        errors = True
                        echo_failure('{}: Missing columns {}'.format(current_check, missing_headers))

                    continue

                # duplicate metric_name
                if row['metric_name'] and row['metric_name'] not in duplicate_set:
                    duplicate_set.add(row['metric_name'])
                else:
                    errors = True
                    echo_failure('{}: `{}` is a duplicate metric_name'.format(current_check, row['metric_name']))

                # metric_name header
                if metric_prefix:
                    if not row['metric_name'].startswith(metric_prefix):
                        prefix = row['metric_name'].split('.')[0]
                        metric_prefix_count[prefix] += 1
                else:
                    errors = True
                    if not metric_prefix_error_shown and current_check not in PROVIDER_INTEGRATIONS:
                        metric_prefix_error_shown = True
                        echo_failure('{}: metric_prefix does not exist in manifest'.format(current_check))

                # metric_type header
                if row['metric_type'] and row['metric_type'] not in VALID_METRIC_TYPE:
                    errors = True
                    echo_failure('{}: `{}` is an invalid metric_type.'.format(current_check, row['metric_type']))

                # unit_name header
                if row['unit_name'] and row['unit_name'] not in VALID_UNIT_NAMES:
                    errors = True
                    echo_failure('{}: `{}` is an invalid unit_name.'.format(current_check, row['unit_name']))

                # orientation header
                if row['orientation'] and row['orientation'] not in VALID_ORIENTATION:
                    errors = True
                    echo_failure('{}: `{}` is an invalid orientation.'.format(current_check, row['orientation']))

                # empty required fields
                for header in REQUIRED_HEADERS:
                    if not row[header]:
                        empty_count[header] += 1

                # empty description field, description is recommended
                if not row['description']:
                    empty_warning_count['description'] += 1
                # exceeds max allowed length of description
                elif len(row['description']) > MAX_DESCRIPTION_LENGTH:
                    errors = True
                    echo_failure(
                        '{}: `{}` exceeds the max length: {} for descriptions.'.format(
                            current_check, row['metric_name'], MAX_DESCRIPTION_LENGTH
                        )
                    )

        for header, count in iteritems(empty_count):
            errors = True
            echo_failure('{}: {} is empty in {} rows.'.format(current_check, header, count))

        for header, count in iteritems(empty_warning_count):
            echo_warning('{}: {} is empty in {} rows.'.format(current_check, header, count))

        for prefix, count in iteritems(metric_prefix_count):
            # Don't spam this warning when we're validating everything
            if check:
                echo_warning(
                    '{}: `{}` appears {} time(s) and does not match metric_prefix '
                    'defined in the manifest.'.format(current_check, prefix, count)
                )

    if errors:
        abort()
626559
def _yarn_app_metrics(self, rm_address, instance, app_tags, addl_tags):
        
        metrics_json = self._rest_request_to_json(
            rm_address, instance, YARN_APPS_PATH, addl_tags, states=YARN_APPLICATION_STATES
        )

        if metrics_json and metrics_json['apps'] is not None and metrics_json['apps']['app'] is not None:

            for app_json in metrics_json['apps']['app']:

                tags = []
                for dd_tag, yarn_key in iteritems(app_tags):
                    try:
                        val = app_json[yarn_key]
                        if val:
                            tags.append('{tag}:{value}'.format(tag=dd_tag, value=val))
                    except KeyError:
                        self.log.error("Invalid value {} for application_tag".format(yarn_key))

                tags.extend(addl_tags)

                self._set_yarn_metrics_from_json(tags, app_json, YARN_APP_METRICS)
1210309
def log_metrics(self, metrics_by_name, info):
        
        try:
            metrics_path = os.path.join(self.dir, "metrics.json")
            with open(metrics_path, 'r') as f:
                saved_metrics = json.load(f)
        except IOError:
            # We haven't recorded anything yet. Start Collecting.
            saved_metrics = {}

        for metric_name, metric_ptr in metrics_by_name.items():

            if metric_name not in saved_metrics:
                saved_metrics[metric_name] = {"values": [],
                                              "steps": [],
                                              "timestamps": []}

            saved_metrics[metric_name]["values"] += metric_ptr["values"]
            saved_metrics[metric_name]["steps"] += metric_ptr["steps"]

            # Manually convert them to avoid passing a datetime dtype handler
            # when we're trying to convert into json.
            timestamps_norm = [ts.isoformat()
                               for ts in metric_ptr["timestamps"]]
            saved_metrics[metric_name]["timestamps"] += timestamps_norm

        self.save_json(saved_metrics, 'metrics.json')
395511
def compute_metrics(self):
        
        for prev, point in pairwise(self.points):
            point.compute_metrics(prev)
        return self
1155798
def _variant_eval_metrics(self, dup_bam):
        
        base, ext = os.path.splitext(dup_bam)
        end_strip = "-dup"
        base = base[:-len(end_strip)] if base.endswith(end_strip) else base
        mfiles = glob.glob("%s*eval_metrics" % base)
        if len(mfiles) > 0:
            with open(mfiles[0]) as in_handle:
                # pull the metrics as JSON from the last line in the file
                for line in in_handle:
                    pass
                metrics = json.loads(line)
            return metrics
        else:
            return None
810321
def _parse_qualimap_metrics(report_file, data):
    
    if not utils.file_exists(report_file):
        return {}
    from bs4 import BeautifulSoup
    out = {}
    parsers = {"Globals": _parse_qualimap_globals,
               "Globals (inside of regions)": _parse_qualimap_globals_inregion,
               "Coverage": _parse_qualimap_coverage,
               "Coverage (inside of regions)": _parse_qualimap_coverage,
               "Insert size": _parse_qualimap_insertsize,
               "Insert size (inside of regions)": _parse_qualimap_insertsize}
    with open(report_file) as in_handle:
        root = BeautifulSoup(in_handle.read(), "html.parser")
    for table in root.find_all("div", class_="table-summary"):
        h3 = table.find("h3")
        if h3.text in parsers:
            out.update(parsers[h3.text](table.find("table")))
    new_names = []
    for metric in out:
        if "qualimap_full" not in tz.get_in(("config", "algorithm", "tools_on"), data, []):
            metric += "_qualimap_1e7reads_est"
        new_names.append(metric)
    out = dict(zip(new_names, out.values()))
    return out
912016
def merge(files, accumulate=True):
        
        metrics = {}
        for f in files:
            parts = os.path.basename(f).split('_')
            typ = parts[0]
            d = MmapedDict(f, read_mode=True)
            for key, value in d.read_all_values():
                metric_name, name, labels = json.loads(key)
                labels_key = tuple(sorted(labels.items()))

                metric = metrics.get(metric_name)
                if metric is None:
                    metric = Metric(metric_name, 'Multiprocess metric', typ)
                    metrics[metric_name] = metric

                if typ == 'gauge':
                    pid = parts[2][:-3]
                    metric._multiprocess_mode = parts[1]
                    metric.add_sample(name, labels_key + (('pid', pid),), value)
                else:
                    # The duplicates and labels are fixed in the next for.
                    metric.add_sample(name, labels_key, value)
            d.close()

        for metric in metrics.values():
            samples = defaultdict(float)
            buckets = {}
            for s in metric.samples:
                name, labels, value = s.name, s.labels, s.value
                if metric.type == 'gauge':
                    without_pid = tuple(l for l in labels if l[0] != 'pid')
                    if metric._multiprocess_mode == 'min':
                        current = samples.setdefault((name, without_pid), value)
                        if value < current:
                            samples[(s.name, without_pid)] = value
                    elif metric._multiprocess_mode == 'max':
                        current = samples.setdefault((name, without_pid), value)
                        if value > current:
                            samples[(s.name, without_pid)] = value
                    elif metric._multiprocess_mode == 'livesum':
                        samples[(name, without_pid)] += value
                    else:  # all/liveall
                        samples[(name, labels)] = value

                elif metric.type == 'histogram':
                    bucket = tuple(float(l[1]) for l in labels if l[0] == 'le')
                    if bucket:
                        # _bucket
                        without_le = tuple(l for l in labels if l[0] != 'le')
                        buckets.setdefault(without_le, {})
                        buckets[without_le].setdefault(bucket[0], 0.0)
                        buckets[without_le][bucket[0]] += value
                    else:
                        # _sum/_count
                        samples[(s.name, labels)] += value

                else:
                    # Counter and Summary.
                    samples[(s.name, labels)] += value

            # Accumulate bucket values.
            if metric.type == 'histogram':
                for labels, values in buckets.items():
                    acc = 0.0
                    for bucket, value in sorted(values.items()):
                        sample_key = (
                            metric.name + '_bucket',
                            labels + (('le', floatToGoString(bucket)),),
                        )
                        if accumulate:
                            acc += value
                            samples[sample_key] = acc
                        else:
                            samples[sample_key] = value
                    if accumulate:
                        samples[(metric.name + '_count', labels)] = acc

            # Convert to correct sample format.
            metric.samples = [Sample(name_, dict(labels), value) for (name_, labels), value in samples.items()]
        return metrics.values()
1182586
def parse_result_line(cls, columns, line, metrics):
        
        line = IORMetricsExtractor.RE_MULTIPLE_SPACES.sub(' ', line)
        line = line.split(' ')
        operation = line[0]
        assert len(line) == len(columns)
        for i, col in enumerate(line):
            col = columns[i]
            if col in cls.METAS_IGNORED:
                continue
            desc = cls.METAS.get(col)
            if desc is None:
                LOGGER.warn('Unrecognized column: %s', col)
            meta = cls.get_meta_name(operation, desc.get('name') or col)
            value = desc['metric'].type(line[i])
            metrics[meta] = value