Take the percentage of contamination from all the parsed *.SELFSM files and add it to the basic stats table at the top of the report
168045
def verifybamid_general_stats_table(self):
		

		# create a dictionary to hold the columns to add to the general stats table
		headers = OrderedDict()
		# available columns are:
		#SEQ_ID RG  CHIP_ID #SNPS   #READS  AVG_DP  FREEMIX FREELK1 FREELK0 FREE_RH FREE_RA CHIPMIX CHIPLK1 CHIPLK0 CHIP_RH CHIP_RA DPREF   RDPHET  RDPALT
		#see https://genome.sph.umich.edu/wiki/VerifyBamID#Interpreting_output_files

		# add the CHIPMIX column if we have data
		if not self.hide_chip_columns:
			headers['CHIPMIX'] = dict(self.col_config_defaults, **{
				'title': 'Contamination (S+A)',
				'description': 'VerifyBamID: CHIPMIX -   Sequence+array estimate of contamination (NA if the external genotype is unavailable)'
			})

		# add the FREEMIX column. set the title and description
		headers['FREEMIX'] = dict(self.col_config_defaults, **{
			'title': 'Contamination (S)',
			'description': 'VerifyBamID: FREEMIX -   Sequence-only estimate of contamination.',
		})

		# pass the data dictionary and header dictionary to function to add to table.
		self.general_stats_addcols(self.verifybamid_data, headers)
1282511
def _run_collect_allelic_counts(pos_file, pos_name, work_dir, data):
    
    out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "structural", "counts"))
    out_file = os.path.join(out_dir, "%s-%s-counts.tsv" % (dd.get_sample_name(data), pos_name))
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            params = ["-T", "CollectAllelicCounts", "-L", pos_file, "-I", dd.get_align_bam(data),
                      "-R", dd.get_ref_file(data), "-O", tx_out_file]
            _run_with_memory_scaling(params, tx_out_file, data)
    return out_file
252339
def report(self):
        
        # Initialise a variable to store the results
        data = ''
        for sample in self.metadata:
            if sample[self.analysistype].primers != 'NA':
                # Set the name of the strain-specific report
                sample[self.analysistype].report = os.path.join(sample[self.analysistype].reportdir,
                                                                '{}_{}.csv'.format(sample.name, self.analysistype))
                # Populate the strain-specific string with header, and strain name
                strainspecific = 'Strain,{},\n{},'.format(','.join(sorted(sample[self.analysistype].targets)),
                                                          sample.name)
                # Iterate through all the genes in the organism-specific analysis
                for gene in sorted(sample[self.analysistype].targets):
                    try:
                        # Extract the percent identity
                        percentidentity = sample[self.analysistype].blastresults[gene]['percent_identity']
                        # If the % identity is greater than the cutoff of 50%, the gene is considered to be present
                        if percentidentity > 50:
                            strainspecific += '{},'.format(percentidentity)
                        else:
                            strainspecific += '-,'
                    # If there are no BLAST results, then the gene is absent
                    except KeyError:
                        strainspecific += '-,'
                strainspecific += '\n'
                # Open and write the data to the strain-specific report
                with open(sample[self.analysistype].report, 'w') as specificreport:
                    specificreport.write(strainspecific)
                # Add all the data from each strain to the cumulative data string
                data += strainspecific
        # Open and write the cumulative data to the cumulative report
        with open(os.path.join(self.reportdir, '{}.csv'.format(self.analysistype)), 'w') as report:
            report.write(data)
545338
def clipandmerge_general_stats_table(self):
        

        headers = OrderedDict()
        headers['percentage'] = {
            'title': '% Merged',
            'description': 'Percentage of reads merged',
            'min': 0,
            'max': 100,
            'suffix': '%',
            'scale': 'Greens',
            'format': '{:,.2f}',
        }
        self.general_stats_addcols(self.clipandmerge_data, headers)
1178128
def make_checksum_validation_script(stats_list):
    """"""
    if not os.path.exists('./hash_check'):
        os.mkdir('./hash_check')

    with open('./hash_check/curl.sh', 'w') as curl_f, open(
        './hash_check/md5.txt', 'w'
    ) as md5_f, open('./hash_check/sha1.txt', 'w') as sha1_f:

        curl_f.write('#!/usr/bin/env bash\n\n')

        for stats_dict in stats_list:
            for sysmeta_xml in stats_dict['largest_sysmeta_xml']:
                print(sysmeta_xml)
                sysmeta_pyxb = d1_common.types.dataoneTypes_v1_2.CreateFromDocument(
                    sysmeta_xml
                )

                pid = sysmeta_pyxb.identifier.value().encode('utf-8')
                file_name = re.sub('\W+', '_', pid)
                size = sysmeta_pyxb.size
                base_url = stats_dict['gmn_dict']['base_url']

                if size > 100 * 1024 * 1024:
                    logging.info('Ignored large object. size={} pid={}')

                curl_f.write('# {} {}\n'.format(size, pid))
                curl_f.write(
                    'curl -o obj/{} {}/v1/object/{}\n'.format(
                        file_name, base_url, d1_common.url.encodePathElement(pid)
                    )
                )

                if sysmeta_pyxb.checksum.algorithm == 'MD5':
                    md5_f.write(
                        '{} obj/{}\n'.format(sysmeta_pyxb.checksum.value(), file_name)
                    )
                else:
                    sha1_f.write(
                        '{} obj/{}\n'.format(sysmeta_pyxb.checksum.value(), file_name)
                    )

    with open('./hash_check/check.sh', 'w') as f:
        f.write('#!/usr/bin/env bash\n\n')
        f.write('mkdir -p obj\n')
        f.write('./curl.sh\n')
        f.write('sha1sum -c sha1.txt\n')
        f.write('md5sum -c md5.txt\n')
208469
def _refresh_stats(self):
        

        # regex identifies all possible stats files.
        #regex = self.outfolder +  "*_stats.tsv"       
        #stats_files = glob.glob(regex)
        #stats_files.insert(self.pipeline_stats_file) # last one is the current pipeline
        #for stats_file in stats_files:

        stats_file = self.pipeline_stats_file
        if os.path.isfile(self.pipeline_stats_file):
            with open(stats_file, 'r') as stat_file:
                for line in stat_file:
                    try:
                        # Someone may have put something that's not 3 columns in the stats file
                        # if so, shame on him, but we can just ignore it.
                        key, value, annotation  = line.split('\t')
                    except ValueError:
                        print("WARNING: Each row in a stats file is expected to have 3 columns")

                    if annotation.rstrip() == self.name or annotation.rstrip() == "shared":
                        self.stats_dict[key] = value.strip()
213618
def report_saved(report_stats):
    """"""
    if Settings.verbose:
        report = ''
        truncated_filename = truncate_cwd(report_stats.final_filename)

        report += '{}: '.format(truncated_filename)
        total = new_percent_saved(report_stats)
        if total:
            report += total
        else:
            report += '0%'
        if Settings.test:
            report += ' could be saved.'
        if Settings.verbose > 1:
            tools_report = ', '.join(report_stats.report_list)
            if tools_report:
                report += '\n\t' + tools_report
        print(report)
37268
def finish_statistics(self):
        
        # params = self.stat.setup_params(self.data)
        self.stat.finish_layer(self.data, self.stat.params)
978316
def analyze(self, output_file=None):
        

        fr = None
        for path, code, line, char, desc in self.parser.parse():

            # Create a new FileResult and register it if we have changed to a new file
            if path not in self.files:
                # Update statistics
                if fr:
                    self.update_stats(fr)
                fr = FileResult(path)
                self.files[path] = fr

            # Add line to the FileResult
            fr.add_error(code, line, char, desc)

        # Add final FileResult to statistics, if any were parsed
        if fr:
            self.update_stats(fr)

        # Generate HTML file
        self.generate(output_file=output_file)
429062
def new_percent_saved(report_stats):
    """"""
    size_in = report_stats.bytes_in
    if size_in > 0:
        size_out = report_stats.bytes_out
        ratio = size_out / size_in
        kb_saved = _humanize_bytes(size_in - size_out)
    else:
        ratio = 0
        kb_saved = 0
    percent_saved = (1 - ratio) * 100

    result = '{:.{prec}f}% ({})'.format(percent_saved, kb_saved, prec=2)
    return result
1005747
def get_stats(self):
    """"""
    ostr = ''
    errtotal = self.deletions['total']+self.insertions['total']+self.mismatches
    ostr += "ALIGNMENT_COUNT\t"+str(self.alignment_count)+"\n"
    ostr += "ALIGNMENT_BASES\t"+str(self.alignment_length)+"\n"
    ostr += "ANY_ERROR\t"+str(errtotal)+"\n"
    ostr += "MISMATCHES\t"+str(self.mismatches)+"\n"
    ostr += "ANY_DELETION\t"+str(self.deletions['total'])+"\n"
    ostr += "COMPLETE_DELETION\t"+str(self.deletions['specific'])+"\n"
    ostr += "HOMOPOLYMER_DELETION\t"+str(self.deletions['homopolymer'])+"\n"
    ostr += "ANY_INSERTION\t"+str(self.insertions['total'])+"\n"
    ostr += "COMPLETE_INSERTION\t"+str(self.insertions['specific'])+"\n"
    ostr += "HOMOPOLYMER_INSERTION\t"+str(self.insertions['homopolymer'])+"\n"
    return ostr
1283703
def conpair_general_stats_table(self):
        

        headers = {}
        headers['concordance_concordance'] = {
            'title': 'Concordance',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'format': '{:,.2f}',
            'scale': 'RdYlGn'
        }
        headers['contamination_normal'] = {
            'title': 'N Contamination',
            'description': 'Normal sample contamination level',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'format': '{:,.3f}',
            'scale': 'RdYlBu-rev'
        }
        headers['contamination_tumor'] = {
            'title': 'T Contamination',
            'description': 'Tumor sample contamination level',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'format': '{:,.3f}',
            'scale': 'RdYlBu-rev'
        }
        self.general_stats_addcols(self.conpair_data, headers)
468959
def add_stats_table(self):
        """"""

        totals = {sample: sum(counts.values())
                  for sample, counts in self.data.items()}

        percentages = {sample: {k: (v / totals[sample]) * 100
                                for k, v in counts.items()}
                       for sample, counts in self.data.items()}

        headers = {
            'species_a': {
                'title': '% Species a',
                'description': 'Percentage of reads mapping to species a',
                'max': 100,
                'min': 0,
                'suffix': '%',
                'scale': 'YlGn'
            }
        }

        self.general_stats_addcols(percentages, headers)
1239434
def dump_stats(self):
        """"""
        log.info('Dumping statistics:')
        stats = []
        stats.append(
            'client-request-errors {}'.format(self._client_request_errors))
        stats.append('denied-allocations {}'.format(self._denied_allocations))
        stats.append('num-ports-managed {}'.format(self._port_pool.num_ports()))
        stats.append('num-ports-checked-for-last-request {}'.format(
            self._port_pool.ports_checked_for_last_request))
        stats.append('total-allocations {}'.format(self._total_allocations))
        for stat in stats:
            log.info(stat)
600598
private void updateStats() {
        EhcacheStats stats = getProducerStats();
        Statistics statistics = underlyingCache.getStatistics();
        double accesses = statistics.getCacheHits() + statistics.getCacheMisses();
        double hitRatio = accesses != 0 ? statistics.getCacheHits() / accesses : 0;

        stats.getStatisticsAccuracy().setValueAsString(statistics.getStatisticsAccuracyDescription());
        stats.getHitRatio().setValueAsDouble(hitRatio);
        stats.getHits().setValueAsLong(statistics.getCacheHits());
        stats.getMisses().setValueAsLong(statistics.getCacheMisses());
        stats.getElements().setValueAsLong(statistics.getObjectCount());

        stats.getInMemoryHits().setValueAsLong(statistics.getInMemoryHits());
        stats.getInMemoryMisses().setValueAsLong(statistics.getInMemoryMisses());
        stats.getInMemoryElements().setValueAsLong(statistics.getMemoryStoreObjectCount());

        stats.getOnDiskHits().setValueAsLong(statistics.getOnDiskHits());
        stats.getOnDiskMisses().setValueAsLong(statistics.getOnDiskMisses());
        stats.getOnDiskElements().setValueAsLong(statistics.getDiskStoreObjectCount());

        stats.getOffHeapHits().setValueAsLong(statistics.getOffHeapHits());
        stats.getOffHeapMisses().setValueAsLong(statistics.getOffHeapMisses());
        stats.getOffHeapElements().setValueAsLong(statistics.getOffHeapStoreObjectCount());

        stats.getAverageGetTime().setValueAsDouble(statistics.getAverageGetTime());
        stats.getAverageSearchTime().setValueAsLong(statistics.getAverageSearchTime());

        stats.getSearchesPerSecond().setValueAsLong(statistics.getSearchesPerSecond());
        stats.getEvictionCount().setValueAsLong(statistics.getEvictionCount());
        stats.getWriterQueueLength().setValueAsLong(statistics.getWriterQueueSize());
    }
699505
def methods(self):
        
        self.contamination_detection()
        ReportImage(self, 'confindr')
        self.run_genesippr()
        ReportImage(self, 'genesippr')
        self.run_sixteens()
        self.run_mash()
        self.run_gdcs()
        ReportImage(self, 'gdcs')
737113
def hisat2_general_stats_table(self):
        

        headers = OrderedDict()
        headers['overall_alignment_rate'] = {
            'title': '% Aligned',
            'description': 'overall alignment rate',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'scale': 'YlGn'
        }
        self.general_stats_addcols(self.hisat2_data, headers)