Find RSeQC junction_saturation frequency reports and parse their data
2688
def parse_reports(self):
    

    # Set up vars
    self.junction_saturation_all = dict()
    self.junction_saturation_known = dict()
    self.junction_saturation_novel = dict()

    # Go through files and parse data
    for f in self.find_log_files('rseqc/junction_saturation'):
        parsed = dict()
        for l in f['f'].splitlines():
            r = re.search(r"^([xyzw])=c\(([\d,]+)\)$", l)
            if r:
                parsed[r.group(1)] = [float(i) for i in r.group(2).split(',')]
        if len(parsed) == 4:
            if parsed['z'][-1] == 0:
                log.warn("Junction saturation data all zeroes, skipping: '{}'".format(f['s_name']))
            else:
                if f['s_name'] in self.junction_saturation_all:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(f['s_name']))
                self.add_data_source(f, section='junction_saturation')
                self.junction_saturation_all[f['s_name']] = OrderedDict()
                self.junction_saturation_known[f['s_name']] = OrderedDict()
                self.junction_saturation_novel[f['s_name']] = OrderedDict()
                for k, v in enumerate(parsed['x']):
                    self.junction_saturation_all[f['s_name']][v] = parsed['z'][k]
                    self.junction_saturation_known[f['s_name']][v] = parsed['y'][k]
                    self.junction_saturation_novel[f['s_name']][v] = parsed['w'][k]

    # Filter to strip out ignored sample names
    self.junction_saturation_all = self.ignore_samples(self.junction_saturation_all)
    self.junction_saturation_known = self.ignore_samples(self.junction_saturation_known)
    self.junction_saturation_novel = self.ignore_samples(self.junction_saturation_novel)

    if len(self.junction_saturation_all) > 0:

        # Add line graph to section
        pconfig = {
            'id': 'rseqc_junction_saturation_plot',
            'title': 'RSeQC: Junction Saturation',
            'ylab': 'Number of Junctions',
            'ymin': 0,
            'xlab': "Percent of reads",
            'xmin': 0,
            'xmax': 100,
            'tt_label': "<strong>{point.x}% of reads</strong>: {point.y:.2f}",
            'data_labels': [
                {'name': 'Known Junctions'},
                {'name': 'Novel Junctions'},
                {'name': 'All Junctions'}
            ],
            'cursor': 'pointer',
            'click_func': plot_single()
        }
        self.add_section (
            name = 'Junction Saturation',
            anchor = 'rseqc-junction_saturation',
            description = '''<a href="http://rseqc.sourceforge.net/#junction-saturation-py" target="_blank">Junction Saturation</a>
                counts the number of known splicing junctions that are observed
                in each dataset. If sequencing depth is sufficient, all (annotated) splice junctions should
                be rediscovered, resulting in a curve that reaches a plateau. Missing low abundance splice
                junctions can affect downstream analysis.</p>
                <div class="alert alert-info" id="rseqc-junction_sat_single_hint">
                  <span class="glyphicon glyphicon-hand-up"></span>
                  Click a line to see the data side by side (as in the original RSeQC plot).
                </div><p>''',
            plot = linegraph.plot([
                    self.junction_saturation_known,
                    self.junction_saturation_novel,
                    self.junction_saturation_all
                ], pconfig)
        )

    # Return number of samples found
    return len(self.junction_saturation_all)
321060
def _check_freqs(parts, indexes):
    
    thresh_ratio = 2.7
    try:  # FreeBayes
        ao_index = parts[8].split(":").index("AO")
        ro_index = parts[8].split(":").index("RO")
    except ValueError:
        ao_index, ro_index = None, None
    try:  # VarDict
        af_index = parts[8].split(":").index("AF")
    except ValueError:
        af_index = None
    if af_index is None and ao_index is None:
        # okay to skip if a gVCF record
        if parts[4].find("<*>") == -1:
            raise NotImplementedError("Unexpected format annotations: %s" % parts[8])
    def _calc_freq(item):
        try:
            if ao_index is not None and ro_index is not None:
                ao = sum([int(x) for x in item.split(":")[ao_index].split(",")])
                ro = int(item.split(":")[ro_index])
                freq = ao / float(ao + ro)
            elif af_index is not None:
                freq = float(item.split(":")[af_index])
            else:
                freq = 0.0
        except (IndexError, ValueError, ZeroDivisionError):
            freq = 0.0
        return freq
    tumor_freq, normal_freq = _calc_freq(parts[indexes["tumor"]]), _calc_freq(parts[indexes["normal"]])
    return normal_freq <= 0.001 or normal_freq <= tumor_freq / thresh_ratio
963691
def _read_frequency_data(self):
        
        temp_dict = read_pattern(
            self.text, {
                "frequencies":
                r"\s*Frequency:\s+([\d\-\.]+)(?:\s+([\d\-\.]+)(?:\s+([\d\-\.]+))*)*",
                "IR_intens":
                r"\s*IR Intens:\s+([\d\-\.]+)(?:\s+([\d\-\.]+)(?:\s+([\d\-\.]+))*)*",
                "IR_active":
                r"\s*IR Active:\s+([YESNO]+)(?:\s+([YESNO]+)(?:\s+([YESNO]+))*)*",
                "ZPE":
                r"\s*Zero point vibrational energy:\s+([\d\-\.]+)\s+kcal/mol",
                "trans_enthalpy":
                r"\s*Translational Enthalpy:\s+([\d\-\.]+)\s+kcal/mol",
                "rot_enthalpy":
                r"\s*Rotational Enthalpy:\s+([\d\-\.]+)\s+kcal/mol",
                "vib_enthalpy":
                r"\s*Vibrational Enthalpy:\s+([\d\-\.]+)\s+kcal/mol",
                "gas_constant":
                r"\s*gas constant \(RT\):\s+([\d\-\.]+)\s+kcal/mol",
                "trans_entropy":
                r"\s*Translational Entropy:\s+([\d\-\.]+)\s+cal/mol\.K",
                "rot_entropy":
                r"\s*Rotational Entropy:\s+([\d\-\.]+)\s+cal/mol\.K",
                "vib_entropy":
                r"\s*Vibrational Entropy:\s+([\d\-\.]+)\s+cal/mol\.K",
                "total_enthalpy":
                r"\s*Total Enthalpy:\s+([\d\-\.]+)\s+kcal/mol",
                "total_entropy":
                r"\s*Total Entropy:\s+([\d\-\.]+)\s+cal/mol\.K"
            })

        keys = ["ZPE", "trans_enthalpy", "rot_enthalpy", "vib_enthalpy", "gas_constant", "trans_entropy", "rot_entropy", "vib_entropy", "total_enthalpy", "total_entropy"]

        for key in keys:
            if temp_dict.get(key) == None:
                self.data[key] = None
            else:
                self.data[key] = float(temp_dict.get(key)[0][0])

        if temp_dict.get('frequencies') == None:
            self.data['frequencies'] = None
            self.data['IR_intens'] = None
            self.data['IR_active'] = None
        else:
            temp_freqs = [
                value for entry in temp_dict.get('frequencies')
                for value in entry
            ]
            temp_intens = [
                value for entry in temp_dict.get('IR_intens')
                for value in entry
            ]
            active = [
                value for entry in temp_dict.get('IR_active')
                for value in entry
            ]
            self.data['IR_active'] = active

            freqs = np.zeros(len(temp_freqs) - temp_freqs.count('None'))
            for ii, entry in enumerate(temp_freqs):
                if entry != 'None':
                    freqs[ii] = float(entry)
            self.data['frequencies'] = freqs

            intens = np.zeros(len(temp_intens) - temp_intens.count('None'))
            for ii, entry in enumerate(temp_intens):
                if entry != 'None':
                    intens[ii] = float(entry)
            self.data['IR_intens'] = intens

            header_pattern = r"\s*Raman Active:\s+[YESNO]+\s+(?:[YESNO]+\s+)*X\s+Y\s+Z\s+(?:X\s+Y\s+Z\s+)*"
            table_pattern = r"\s*[a-zA-Z][a-zA-Z\s]\s*([\d\-\.]+)\s*([\d\-\.]+)\s*([\d\-\.]+)\s*(?:([\d\-\.]+)\s*([\d\-\.]+)\s*([\d\-\.]+)\s*(?:([\d\-\.]+)\s*([\d\-\.]+)\s*([\d\-\.]+))*)*"
            footer_pattern = r"TransDip\s+[\d\-\.]+\s*[\d\-\.]+\s*[\d\-\.]+\s*(?:[\d\-\.]+\s*[\d\-\.]+\s*[\d\-\.]+\s*)*"
            temp_freq_mode_vecs = read_table_pattern(
                self.text, header_pattern, table_pattern, footer_pattern)
            freq_mode_vecs = np.zeros(
                shape=(len(freqs), len(temp_freq_mode_vecs[0]), 3))

            for ii, triple_FMV in enumerate(temp_freq_mode_vecs):
                for jj, line in enumerate(triple_FMV):
                    for kk, entry in enumerate(line):
                        if entry != 'None':
                            freq_mode_vecs[int(ii * 3 + math.floor(kk / 3)),
                                           jj, kk % 3] = float(entry)

            self.data["frequency_mode_vectors"] = freq_mode_vecs
587189
def parsebp(self, bpfile, debug=False):
        

        # bandpass. taking liberally from Corder et al's analysisutilities
        ([polyMode, polyType, nPolyAmp, nPolyPhase, scaleFactor, nRows, nSpws, nUniqueTimesBP, uniqueTimesBP,
          nPolarizations, frequencyLimits, increments, frequenciesGHz, polynomialPhase,
          polynomialAmplitude, timesBP, antennasBP, cal_desc_idBP, spwBP]) = openBpolyFile(bpfile, debug)

        # index iterates over antennas, then times/sources (solution sets). each index has 2x npoly, which are 2 pols
        polynomialAmplitude = n.array(polynomialAmplitude); polynomialPhase = n.array(polynomialPhase)
        polynomialAmplitude[:,0] = 0.; polynomialAmplitude[:,nPolyAmp] = 0.
        polynomialPhase[:,0] = 0.; polynomialPhase[nPolyPhase] = 0.
        ampSolR, ampSolL = calcChebyshev(polynomialAmplitude, frequencyLimits, n.array(frequenciesGHz)*1e+9)
        phaseSolR, phaseSolL = calcChebyshev(polynomialPhase, frequencyLimits, n.array(frequenciesGHz)*1e+9)

        nants = len(n.unique(antennasBP))
        self.bptimes = n.array(timesBP)
        ptsperspec = 1000
        npol = 2
        self.logger.info('Parsed bp solutions for %d solutions, %d ants, %d spw, and %d pols' % (nUniqueTimesBP, nants, nSpws, nPolarizations))
        self.bandpass = n.zeros( (nants, nSpws*ptsperspec, npol), dtype='complex')
        for spw in range(nSpws):
            ampSolR[spw*nants:(spw+1)*nants] += 1 - ampSolR[spw*nants:(spw+1)*nants].mean()     # renormalize mean over ants (per spw) == 1
            ampSolL[spw*nants:(spw+1)*nants] += 1 - ampSolL[spw*nants:(spw+1)*nants].mean()
            for ant in range(nants):
                self.bandpass[ant, spw*ptsperspec:(spw+1)*ptsperspec, 0] = ampSolR[ant+spw*nants] * n.exp(1j*phaseSolR[ant+spw*nants])
                self.bandpass[ant, spw*ptsperspec:(spw+1)*ptsperspec, 1] = ampSolL[ant+spw*nants] * n.exp(1j*phaseSolL[ant+spw*nants])

        self.bpfreq = n.zeros( (nSpws*ptsperspec) )
        for spw in range(nSpws):
            self.bpfreq[spw*ptsperspec:(spw+1)*ptsperspec] = 1e9 * frequenciesGHz[nants*spw]
330556
def get_bss_load(_, data):
    
    answers = {
        'station count': (data[1] << 8) | data[0],
        'channel utilisation': data[2] / 255.0,
        'available admission capacity': (data[4] << 8) | data[3],
    }
    return answers
388769
def parse_star_report (self, raw_data):
        

        regexes = {
            'total_reads':                  r"Number of input reads \|\s+(\d+)",
            'avg_input_read_length':        r"Average input read length \|\s+([\d\.]+)",
            'uniquely_mapped':              r"Uniquely mapped reads number \|\s+(\d+)",
            'uniquely_mapped_percent':      r"Uniquely mapped reads % \|\s+([\d\.]+)",
            'avg_mapped_read_length':       r"Average mapped length \|\s+([\d\.]+)",
            'num_splices':                  r"Number of splices: Total \|\s+(\d+)",
            'num_annotated_splices':        r"Number of splices: Annotated \(sjdb\) \|\s+(\d+)",
            'num_GTAG_splices':             r"Number of splices: GT/AG \|\s+(\d+)",
            'num_GCAG_splices':             r"Number of splices: GC/AG \|\s+(\d+)",
            'num_ATAC_splices':             r"Number of splices: AT/AC \|\s+(\d+)",
            'num_noncanonical_splices':     r"Number of splices: Non-canonical \|\s+(\d+)",
            'mismatch_rate':                r"Mismatch rate per base, % \|\s+([\d\.]+)",
            'deletion_rate':                r"Deletion rate per base \|\s+([\d\.]+)",
            'deletion_length':              r"Deletion average length \|\s+([\d\.]+)",
            'insertion_rate':               r"Insertion rate per base \|\s+([\d\.]+)",
            'insertion_length':             r"Insertion average length \|\s+([\d\.]+)",
            'multimapped':                  r"Number of reads mapped to multiple loci \|\s+(\d+)",
            'multimapped_percent':          r"% of reads mapped to multiple loci \|\s+([\d\.]+)",
            'multimapped_toomany':          r"Number of reads mapped to too many loci \|\s+(\d+)",
            'multimapped_toomany_percent':  r"% of reads mapped to too many loci \|\s+([\d\.]+)",
            'unmapped_mismatches_percent':  r"% of reads unmapped: too many mismatches \|\s+([\d\.]+)",
            'unmapped_tooshort_percent':    r"% of reads unmapped: too short \|\s+([\d\.]+)",
            'unmapped_other_percent':       r"% of reads unmapped: other \|\s+([\d\.]+)",
        }
        parsed_data = {}
        for k, r in regexes.items():
            r_search = re.search(r, raw_data, re.MULTILINE)
            if r_search:
                parsed_data[k] = float(r_search.group(1))
        # Figure out the numbers for unmapped as for some reason only the percentages are given
        try:
            total_mapped = parsed_data['uniquely_mapped'] + parsed_data['multimapped'] + parsed_data['multimapped_toomany']
            unmapped_count = parsed_data['total_reads'] - total_mapped
            total_unmapped_percent = parsed_data['unmapped_mismatches_percent'] + parsed_data['unmapped_tooshort_percent'] + parsed_data['unmapped_other_percent']
            try:
                parsed_data['unmapped_mismatches'] = int(round(unmapped_count * (parsed_data['unmapped_mismatches_percent'] / total_unmapped_percent), 0))
                parsed_data['unmapped_tooshort'] = int(round(unmapped_count * (parsed_data['unmapped_tooshort_percent'] / total_unmapped_percent), 0))
                parsed_data['unmapped_other'] = int(round(unmapped_count * (parsed_data['unmapped_other_percent'] / total_unmapped_percent), 0))
            except ZeroDivisionError:
                parsed_data['unmapped_mismatches'] = 0
                parsed_data['unmapped_tooshort'] = 0
                parsed_data['unmapped_other'] = 0
        except KeyError:
            pass

        if len(parsed_data) == 0: return None
        return parsed_data
885308
def _parse(self, registers):
        """"""
        result = {'ip': self.ip, 'connected': True}
        decoder = BinaryPayloadDecoder.fromRegisters(registers,
                                                     byteorder=Endian.Big,
                                                     wordorder=Endian.Little)
        # Register 40001 is a collection of alarm status signals
        b = [decoder.decode_bits(), decoder.decode_bits()]
        reg_40001 = b[1] + b[0]
        # Bits 0-3 map to the monitor state
        monitor_integer = sum(1 << i for i, b in enumerate(reg_40001[:4]) if b)
        result['state'] = options['monitor state'][monitor_integer]
        # Bits 4-5 map to fault status
        fault_integer = sum(1 << i for i, b in enumerate(reg_40001[4:6]) if b)
        result['fault'] = {'status': options['fault status'][fault_integer]}
        # Bits 6 and 7 tell if low and high alarms are active
        low, high = reg_40001[6:8]
        result['alarm'] = options['alarm level'][low + high]
        # Bits 8-10 tell if internal sensor relays 1-3 are energized. Skipping.
        # Bit 11 is a heartbeat bit that toggles every two seconds. Skipping.
        # Bit 12 tells if relays are under modbus control. Skipping.
        # Remaining bits are empty. Skipping.
        # Register 40002 has a gas ID and a sensor cartridge ID. Skipping.
        decoder._pointer += 2
        # Registers 40003-40004 are the gas concentration as a float
        result['concentration'] = decoder.decode_32bit_float()
        # Register 40005 is the concentration as an int. Skipping.
        decoder._pointer += 2
        # Register 40006 is the number of the most important fault.
        fault_number = decoder.decode_16bit_uint()
        if fault_number != 0:
            code = ('m' if fault_number < 30 else 'F') + str(fault_number)
            result['fault']['code'] = code
            result['fault'].update(faults[code])
        # Register 40007 holds the concentration unit in the second byte
        # Instead of being an int, it's the position of the up bit
        unit_bit = decoder.decode_bits().index(True)
        result['units'] = options['concentration unit'][unit_bit]
        decoder._pointer += 1
        # Register 40008 holds the sensor temperature in Celsius
        result['temperature'] = decoder.decode_16bit_int()
        # Register 40009 holds number of hours remaining in cell life
        result['life'] = decoder.decode_16bit_uint() / 24.0
        # Register 40010 holds the number of heartbeats (16 LSB). Skipping.
        decoder._pointer += 2
        # Register 40011 is the sample flow rate in cc / min
        result['flow'] = decoder.decode_16bit_uint()
        # Register 40012 is blank. Skipping.
        decoder._pointer += 2
        # Registers 40013-40016 are the alarm concentration thresholds
        result['low-alarm threshold'] = round(decoder.decode_32bit_float(), 6)
        result['high-alarm threshold'] = round(decoder.decode_32bit_float(), 6)
        # Despite what the manual says, thresholds are always reported in ppm.
        # Let's fix that to match the concentration units.
        if result['units'] == 'ppb':
            result['concentration'] *= 1000
            result['low-alarm threshold'] *= 1000
            result['high-alarm threshold'] *= 1000
        return result
979350
def _parse_radic_header(header_group, dipole_mode="all"):
    
    header = ''.join(header_group[1])

    nr_of_electrodes = _get_nr_of_electrodes(header.split('\n'))
    groups = itertools.groupby(
        header.split('\n'),
        lambda line: (
            line.startswith('[End Readings]') or
            line.startswith('[Begin Readings]')
        )
    )

    for key, group in groups:
        # there is only one group we are interested in: the first one
        if key:
            tmp = np.array(
                [
                    np.fromstring(
                        x, sep=' ', dtype=int
                    )
                    for x in next(groups)[1]
                ],
            )

            # curpots = tmp.T.copy()
            # curpots.resize(nr_of_electrodes + 2, tmp.shape[0])
            break

    # now we have the raw readings, pad them with zeros
    readings = []
    for raw_reading in tmp:
        voltage_rus = raw_reading[3:]
        # note that we now have the last electrode in here, too (so one more
        # than we have RUs. This electrode is always 0
        normed_reading = np.zeros(nr_of_electrodes)
        for i in voltage_rus[np.where(voltage_rus > 0)]:
            normed_reading[i - 1] = i
        readings.append(np.hstack((raw_reading[0:3], normed_reading)))

    # now generate the available configurations
    # add calibration
    reading_configs = {}
    for nr, reading in enumerate(readings):
        # nr = reading[0]
        A = reading[1]
        B = reading[2]

        voltages = []

        # loop over the RU-settings
        # we always measure to the next zero electrode
        for i in range(3, nr_of_electrodes + 3):
            for j in range(i + 1, nr_of_electrodes + 3):
                if reading[j] == 0:
                    M = i - 2
                    N = j - 2
                    all_active = ((reading[i] == 0) and (reading[j] == 0))
                    # print('a', 'b', 'm, n', A, B, M, N, all_active)
                    voltages.append((A, B, M, N, all_active))
                    break

        reading_configs[nr + 1] = np.array(voltages)
    return reading_configs
188837
def parse_reports(self):
    

    # Set up vars
    self.bam_stat_data = dict()
    regexes = {
        'total_records': r"Total records:\s*(\d+)",
        'qc_failed': r"QC failed:\s*(\d+)",
        'optical_pcr_duplicate': r"Optical/PCR duplicate:\s*(\d+)",
        'non_primary_hits': r"Non primary hits\s*(\d+)",
        'unmapped_reads': r"Unmapped reads:\s*(\d+)",
        'mapq_lt_mapq_cut_non-unique': r"mapq < mapq_cut \(non-unique\):\s*(\d+)",
        'mapq_gte_mapq_cut_unique': r"mapq >= mapq_cut \(unique\):\s*(\d+)",
        'read_1': r"Read-1:\s*(\d+)",
        'read_2': r"Read-2:\s*(\d+)",
        'reads_map_to_sense': r"Reads map to '\+':\s*(\d+)",
        'reads_map_to_antisense': r"Reads map to '-':\s*(\d+)",
        'non-splice_reads': r"Non-splice reads:\s*(\d+)",
        'splice_reads': r"Splice reads:\s*(\d+)",
        'reads_mapped_in_proper_pairs': r"Reads mapped in proper pairs:\s*(\d+)",
        'proper-paired_reads_map_to_different_chrom': r"Proper-paired reads map to different chrom:\s*(\d+)",
    }

    #intiate PE check
    is_paired_end=False

    # Go through files and parse data using regexes
    for f in self.find_log_files('rseqc/bam_stat'):
        d = dict()
        for k, r in regexes.items():
            r_search = re.search(r, f['f'], re.MULTILINE)
            if r_search:
                d[k] = int(r_search.group(1))

        # Calculate some percentages
        if 'total_records' in d:
            t = float(d['total_records'])
            if 'mapq_gte_mapq_cut_unique' in d:
                d['unique_percent'] = (float(d['mapq_gte_mapq_cut_unique']) / t)*100.0
            if 'reads_mapped_in_proper_pairs' in d:
                d['proper_pairs_percent'] = (float(d['reads_mapped_in_proper_pairs']) / t)*100.0

        if len(d) > 0:
            if f['s_name'] in self.bam_stat_data:
                log.debug("Duplicate sample name found! Overwriting: {}".format(f['s_name']))
            self.add_data_source(f, section='bam_stat')
            #Check if SE or PE
            if d['read_2'] != 0:
                is_paired_end = True
            self.bam_stat_data[f['s_name']] = d

    # Filter to strip out ignored sample names
    self.bam_stat_data = self.ignore_samples(self.bam_stat_data)

    if len(self.bam_stat_data) > 0:
        # Write to file
        self.write_data_file(self.bam_stat_data, 'multiqc_rseqc_bam_stat')

        # Add to general stats table
        self.general_stats_headers['proper_pairs_percent'] = {
            'title': '% Proper Pairs',
            'description': '% Reads mapped in proper pairs',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'scale': 'RdYlGn'
        }
        for s_name in self.bam_stat_data:
            if s_name not in self.general_stats_data:
                self.general_stats_data[s_name] = dict()

                #Only write if PE, i.e. there is something to write
                if is_paired_end:
                    self.general_stats_data[s_name].update( self.bam_stat_data[s_name] )

        # Make dot plot of counts
        pconfig = {
            'id': 'rseqc_bam_stat'
        }
        keys = OrderedDict()
        defaults = {
            'min': 0,
            'shared_key': 'read_count',
            'decimalPlaces': 2,
            'modify': lambda x: float(x) / 1000000.0,
        }
        keys['total_records'] = dict(defaults, **{'title': 'Total records' })
        keys['qc_failed'] = dict(defaults, **{'title': 'QC failed' })
        keys['optical_pcr_duplicate'] = dict(defaults, **{'title': 'Duplicates', 'description': 'Optical/PCR duplicate' })
        keys['non_primary_hits'] = dict(defaults, **{'title': 'Non primary hit' })
        keys['unmapped_reads'] = dict(defaults, **{'title': 'Unmapped', 'description': 'Unmapped reads' })
        keys['mapq_lt_mapq_cut_non'] = dict(defaults, **{'title': 'Non-unique', 'description': 'mapq < mapq_cut (non-unique)' })
        keys['mapq_gte_mapq_cut_unique'] = dict(defaults, **{'title': 'Unique', 'description': 'mapq >= mapq_cut (unique)' })
        if is_paired_end:
            keys['read_1'] = dict(defaults, **{'title': 'Read-1' })
            keys['read_2'] = dict(defaults, **{'title': 'Read-2' })
        keys['reads_map_to_sense'] = dict(defaults, **{'title': '+ve strand', 'description': "Reads map to '+'" })
        keys['reads_map_to_antisense'] = dict(defaults, **{'title': '-ve strand', 'description': "Reads map to '-'" })
        keys['non-splice_reads'] = dict(defaults, **{'title': 'Non-splice reads' })
        keys['splice_reads'] = dict(defaults, **{'title': 'Splice reads' })
        if is_paired_end:
            keys['reads_mapped_in_proper_pairs'] = dict(defaults, **{
                        'title': 'Proper pairs', 'description':'Reads mapped in proper pairs' })
            keys['proper-paired_reads_map_to_different_chrom'] = dict(defaults, **{
                        'title': 'Different chrom', 'description': 'Proper-paired reads map to different chrom' })

        self.add_section (
            name = 'Bam Stat',
            anchor = 'rseqc-bam_stat',
            description = 'All numbers reported in millions.',
            plot = beeswarm.plot(self.bam_stat_data, keys, pconfig)
        )

    # Return number of samples found
    return len(self.bam_stat_data)
876677
def check_frequencies(pfeed, *, as_df=False, include_warnings=False):
    
    table = 'frequencies'
    problems = []

    # Preliminary checks
    if pfeed.frequencies is None:
        problems.append(['error', 'Missing table', table, []])
    else:
        f = pfeed.frequencies.copy()
        problems = check_for_required_columns(problems, table, f)
    if problems:
        return gt.format_problems(problems, as_df=as_df)

    if include_warnings:
        problems = check_for_invalid_columns(problems, table, f)

    # Check route_short_name and route_long_name
    for column in ['route_short_name', 'route_long_name']:
        problems = gt.check_column(problems, table, f, column, gt.valid_str,
          column_required=False)

    cond = ~(f['route_short_name'].notnull() | f['route_long_name'].notnull())
    problems = gt.check_table(problems, table, f, cond,
      'route_short_name and route_long_name both empty')

    # Check route_type
    v = lambda x: x in range(8)
    problems = gt.check_column(problems, table, f, 'route_type', v)

    # Check service window ID
    problems = gt.check_column_linked_id(problems, table, f,
      'service_window_id', pfeed.service_windows)

    # Check direction
    v = lambda x: x in range(3)
    problems = gt.check_column(problems, table, f, 'direction', v)

    # Check frequency
    v = lambda x: isinstance(x, int)
    problems = gt.check_column(problems, table, f, 'frequency', v)

    # Check speed
    problems = gt.check_column(problems, table, f, 'speed', valid_speed,
      column_required=False)

    # Check shape ID
    problems = gt.check_column_linked_id(problems, table, f, 'shape_id',
      pfeed.shapes)

    return gt.format_problems(problems, as_df=as_df)
1119358
def htseq_stats_table(self):
        

        headers = OrderedDict()
        headers['percent_assigned'] = {
            'title': '% Assigned',
            'description': '% Assigned reads',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'scale': 'RdYlGn'
        }
        headers['assigned'] = {
            'title': '{} Assigned'.format(config.read_count_prefix),
            'description': 'Assigned Reads ({})'.format(config.read_count_desc),
            'min': 0,
            'scale': 'PuBu',
            'modify': lambda x: float(x) * config.read_count_multiplier,
            'shared_key': 'read_count'
        }
        self.general_stats_addcols(self.htseq_data, headers)
1183628
def _parse_sensor_record(self, sensor_data_rec, sensor_info, rem_values):
        
        val_idx = 0

        # @TODO seems there is only a single field in each of these
        assert len(sensor_data_rec.field) == 1
        sensor_data_array = sensor_data_rec.field[0].content

        # there is probably not going to be a count in the def, it'll be in the data
        count = None
        count_text = sensor_data_array.elementCount.text
        if count_text:
            count = int(count_text.strip())

        if not count:
            count = int(rem_values[val_idx])
            val_idx += 1

        parsed = []

        for recnum in range(count):
            cur = []

            for f in sensor_data_array.elementType.field:
                cur_val = rem_values[val_idx]
                val_idx += 1

                m = Member(name=f.name, standard=f.content.definition)

                if hasattr(f.content, "uom"):
                    m["units"] = f.content.uom

                try:
                    m["value"] = float(cur_val)
                except ValueError:
                    m["value"] = cur_val

                if len(f.quality):
                    m["quality"] = []
                    for qual in f.quality:
                        cur_qual = rem_values[val_idx]
                        val_idx += 1

                        # @TODO check this against constraints
                        m["quality"].append(cur_qual)

                cur.append(m)

            parsed.append(cur)

        return parsed, val_idx
397570
def _parse_v2_advertisement(self, rssi, sender, data):
        

        if len(data) != 31:
            return

        # We have already verified that the device is an IOTile device
        # by checking its service data uuid in _process_scan_event so
        # here we just parse out the required information

        device_id, reboot_low, reboot_high_packed, flags, timestamp, \
        battery, counter_packed, broadcast_stream_packed, broadcast_value, \
        _mac = unpack("<LHBBLBBHLL", data[7:])

        reboots = (reboot_high_packed & 0xF) << 16 | reboot_low
        counter = counter_packed & ((1 << 5) - 1)
        broadcast_multiplex = counter_packed >> 5
        broadcast_toggle = broadcast_stream_packed >> 15
        broadcast_stream = broadcast_stream_packed & ((1 << 15) - 1)

        # Flags for version 2 are:
        #   bit 0: Has pending data to stream
        #   bit 1: Low voltage indication
        #   bit 2: User connected
        #   bit 3: Broadcast data is encrypted
        #   bit 4: Encryption key is device key
        #   bit 5: Encryption key is user key
        #   bit 6: broadcast data is time synchronized to avoid leaking
        #   information about when it changes

        self._device_scan_counts.setdefault(device_id, {'v1': 0, 'v2': 0})['v2'] += 1

        info = {'connection_string': sender,
                'uuid': device_id,
                'pending_data': bool(flags & (1 << 0)),
                'low_voltage': bool(flags & (1 << 1)),
                'user_connected': bool(flags & (1 << 2)),
                'signal_strength': rssi,
                'reboot_counter': reboots,
                'sequence': counter,
                'broadcast_toggle': broadcast_toggle,
                'timestamp': timestamp,
                'battery': battery / 32.0,
                'advertising_version':2}

        self._trigger_callback('on_scan', self.id, info, self.ExpirationTime)

        # If there is a valid reading on the advertising data, broadcast it
        if broadcast_stream != 0xFFFF & ((1 << 15) - 1):
            if self._throttle_broadcast and \
               self._check_update_seen_broadcast(sender, timestamp, broadcast_stream, broadcast_value,
                                                 broadcast_toggle, counter=counter, channel=broadcast_multiplex):
                return

            reading = IOTileReading(timestamp, broadcast_stream, broadcast_value, reading_time=datetime.datetime.utcnow())
            report = BroadcastReport.FromReadings(info['uuid'], [reading], timestamp)
            self._trigger_callback('on_report', None, report)
1337108
def get_refresh_stats(self):
        
        if self.rf_draw_count == 0:
            fps = 0.0
        else:
            interval = time.time() - self.rf_start_time
            fps = self.rf_draw_count / interval

        jitter = self.rf_delta_total / max(1, self.rf_timer_count)

        late_avg = self.rf_late_total / max(1, self.rf_late_count)
        late_pct = self.rf_late_count / max(1.0, float(self.rf_timer_count)) * 100

        early_avg = self.rf_early_total / max(1, self.rf_early_count)
        early_pct = self.rf_early_count / max(1.0, float(self.rf_timer_count)) * 100

        balance = self.rf_late_total - self.rf_early_total

        stats = dict(fps=fps, jitter=jitter,
                     early_avg=early_avg, early_pct=early_pct,
                     late_avg=late_avg, late_pct=late_pct,
                     balance=balance)
        return stats
1288814
def parse_from_rsb(header: bytearray) -> dict:
    
    params = {}

    params["text_header_size"] = struct.unpack('I', header[0:4])[0]

    params["events_num"] = struct.unpack('i', header[8:12])[0]

    start_time = struct.unpack('Q', header[16:24])[0]
    params["start_time"] = datetime.fromtimestamp(start_time).isoformat()
    end_time = struct.unpack('Q', header[24:32])[0]
    params["end_time"] = datetime.fromtimestamp(end_time).isoformat()

    params["filepath"] = header[32: 32 + 255].rstrip(b'\0').decode("cp1251")

    params["num_blocks"] = struct.unpack('i', header[288:292])[0]

    params["aquisition_time"] = struct.unpack('i', header[292:296])[0]

    params["blocks_in_file"] = struct.unpack('i', header[296:300])[0]

    params["waitTime"] = struct.unpack('i', header[300:304])[0]

    params["threshold"] = struct.unpack('d', header[312:320])[0]

    sync_params_num = struct.unpack('I', header[336:340])[0]
    sync_params = np.unique(np.frombuffer(header[320:336], np.uint32)
                            [:sync_params_num])
    params["synchro_control"] = []
    for param in sync_params:
        if param == 0:
            params["synchro_control"].append("Default")
        else:
            params["synchro_control"].append(list(synchro_control.keys())
                                             [list(synchro_control.values())
                                              .index(param)])

    params["sample_freq"] = struct.unpack('d', header[344:352])[0]
    params["pre_history"] = struct.unpack('I', header[352:356])[0]

    params["packet_number"] = struct.unpack('i', header[356:360])[0]

    params["b_size"] = struct.unpack('I', header[360:364])[0]

    params["hysteresis"] = struct.unpack('I', header[364:368])[0]

    params["channel_number"] = struct.unpack('I', header[368:372])[0]

    ch_params = []

    for i in range(params["channel_number"]):
        ch_data = header[372 + 56 * i: 372 + 56 * (i + 1)]
        ch_param = {}

        param_num = struct.unpack('I', ch_data[36:40])[0]
        ch_params_raw = np.unique(np.frombuffer(ch_data[4:36],
                                                np.uint32)[:param_num])
        ch_param["params"] = []
        for param in ch_params_raw:
            if param == 0:
                ch_param["params"].append("Default")
            else:
                ch_param["params"].append(list(channel_control.keys())
                                          [list(channel_control.values())
                                           .index(param)])

        ch_param["adjustment"] = struct.unpack('d', ch_data[44:52])[0]
        ch_param["gain"] = struct.unpack('I', ch_data[52:56])[0]
        ch_params.append(ch_param)

    params["channel"] = ch_params

    synchro_channel = {}
    sync_ch_par_num = struct.unpack('I', header[632:636])[0]
    sync_ch_params_raw = np.unique(np.frombuffer(header[600:632],
                                                 np.uint32)[:sync_ch_par_num])

    synchro_channel["params"] = []
    for param in sync_ch_params_raw:
        if param == 0:
            synchro_channel["params"].append("Default")
        else:
            synchro_channel["params"] \
                .append(list(synchro_channel_control.keys())
                        [list(synchro_channel_control.values()).index(param)])

    synchro_channel_type = struct.unpack('I', header[304:308])[0]
    synchro_channel["type"] = list(synchro_channel_types.keys())[
        list(synchro_channel_types.values()).index(synchro_channel_type)]

    synchro_channel["gain"] = struct.unpack('I', header[636:640])[0]

    params["synchro_channel"] = synchro_channel

    params["err_lang"] = struct.unpack('I', header[640:644])[0]
    params["board_name"] = header[644:644 + 255].rstrip(b'\0').decode("cp1251")

    params["board_id"] = struct.unpack('I', header[900:904])[0]

    return params
466547
def parse_reports(self):
    

    # Set up vars
    self.read_gc = dict()
    self.read_gc_pct = dict()

    # Go through files and parse data
    for f in self.find_log_files('rseqc/read_gc'):

        if f['f'].startswith('GC%	read_count'):
            gc = list()
            counts = list()
            for l in f['f'].splitlines():
                s = l.split()
                try:
                    gc.append(float(s[0]))
                    counts.append(float(s[1]))
                except:
                    pass
            if len(gc) > 0:
                sorted_gc_keys = sorted(range(len(gc)), key=lambda k: gc[k])
                total = sum(counts)
                if f['s_name'] in self.read_gc:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(f['s_name']))
                self.add_data_source(f, section='read_GC')
                self.read_gc[f['s_name']] = OrderedDict()
                self.read_gc_pct[f['s_name']] = OrderedDict()
                for i in sorted_gc_keys:
                    self.read_gc[f['s_name']][gc[i]] = counts[i]
                    self.read_gc_pct[f['s_name']][gc[i]] = (counts[i]/total)*100

    # Filter to strip out ignored sample names
    self.read_gc = self.ignore_samples(self.read_gc)

    if len(self.read_gc) > 0:

        # Add line graph to section
        pconfig = {
            'id': 'rseqc_read_gc_plot',
            'title': 'RSeQC: Read GC Content',
            'ylab': 'Number of Reads',
            'xlab': "GC content (%)",
            'xmin': 0,
            'xmax': 100,
            'tt_label': "<strong>{point.x}% GC</strong>: {point.y:.2f}",
            'data_labels': [
                {'name': 'Counts', 'ylab': 'Number of Reads'},
                {'name': 'Percentages', 'ylab': 'Percentage of Reads'}
            ]
        }
        self.add_section (
            name = 'Read GC Content',
            anchor = 'rseqc-read_gc',
            description = '<a href="http://rseqc.sourceforge.net/#read-gc-py" target="_blank">read_GC</a>' \
                " calculates a histogram of read GC content.</p>",
            plot = linegraph.plot([self.read_gc, self.read_gc_pct], pconfig)
        )

    # Return number of samples found
    return len(self.read_gc)
317964
def calc_uncorrelated_signal_power(params):
    
    ''''''


    for i, data_type in enumerate(['LFP','CSD']):
        if i % SIZE == RANK:
    
            # Determine size of PSD matrix
    
            f = h5py.File(os.path.join(params.savefolder, data_type + 'sum.h5'),'r')
            data = f['data'].value
            srate = f['srate'].value
            if ana_params.mlab:
                Psum, freqs = plt.mlab.psd(data[0], NFFT=ana_params.NFFT, Fs=srate,
                                    noverlap=ana_params.noverlap, window=ana_params.window)
            else:
                [freqs, Psum] = hlp.powerspec([data[0]], tbin= 1./srate*1000.,
                                       Df=ana_params.Df, pointProcess=False)
            f.close()
            P = np.zeros((data.shape[0],Psum.shape[0]))
               
            for y in params.y:
    
                print 'processing ', y
                # Load data
                f = h5py.File(os.path.join(params.populations_path, '%s_%ss' %
                                           (y,data_type) + '.h5'),'r')
                data_y = f['data'].value[:,:, ana_params.transient:]
                # subtract mean
                for j in range(len(data_y)):
                    data_yT = data_y[j].T - data_y[j].mean(axis=1)
                    data_y[j] = data_yT.T
                srate = f['srate'].value
                tvec = np.arange(data_y.shape[2]) * 1000. / srate
                f.close()
    
            
                for j in range(len(data_y)): # loop over cells
                    if ana_params.mlab:
                        for ch in range(len(params.electrodeParams['z'])): # loop over channels
                            P_j_ch, freqs = plt.mlab.psd(data_y[j,ch],
                                                         NFFT=ana_params.NFFT, Fs=srate,
                                                         noverlap=ana_params.noverlap,
                                                         window=ana_params.window)
                                       
                            P[ch] += P_j_ch
                    else:
                        [freqs, P_j] = hlp.powerspec(data_y[j], tbin= 1./srate*1000.,
                                                     Df=ana_params.Df, pointProcess=False)
                        mask = np.where(freqs >= 0.)
                        freqs = freqs[mask]
                        P_j = P_j[:,mask][:,0,:]
                        P_j = P_j/tvec[tvec >= ana_params.transient].size**2
                        
                        P += P_j
    
            #rescale PSD as they may be computed from a fraction of single cell LFPs
            P /= params.recordSingleContribFrac
            
            # Save data
            f = h5py.File(os.path.join(params.savefolder, ana_params.analysis_folder,
                                       data_type +  ana_params.fname_psd_uncorr),'w')
            f['freqs']=freqs
            f['psd']=P
            f['transient']=ana_params.transient
            f['mlab']=ana_params.mlab
            f['NFFT']=ana_params.NFFT
            f['noverlap']=ana_params.noverlap
            f['window']=str(ana_params.window)
            f['Df']=str(ana_params.Df)        
            f.close()

    return