Parse the contents of the Qualimap BamQC genome_results.txt file
164845
def parse_genome_results(self, f):
    
    regexes = {
        'bam_file': r"bam file = (.+)",
        'total_reads': r"number of reads = ([\d,]+)",
        'mapped_reads': r"number of mapped reads = ([\d,]+)",
        'mapped_bases': r"number of mapped bases = ([\d,]+)",
        'sequenced_bases': r"number of sequenced bases = ([\d,]+)",
        'mean_insert_size': r"mean insert size = ([\d,\.]+)",
        'median_insert_size': r"median insert size = ([\d,\.]+)",
        'mean_mapping_quality': r"mean mapping quality = ([\d,\.]+)",
        'general_error_rate': r"general error rate = ([\d,\.]+)",
    }
    d = dict()
    for k, r in regexes.items():
        r_search = re.search(r, f['f'], re.MULTILINE)
        if r_search:
            try:
                d[k] = float(r_search.group(1).replace(',',''))
            except ValueError:
                d[k] = r_search.group(1)
    # Check we have an input filename
    if 'bam_file' not in d:
        log.debug("Couldn't find an input filename in genome_results file {}".format(f['fn']))
        return None

    # Get a nice sample name
    s_name = self.clean_s_name(d['bam_file'], f['root'])

    # Add to general stats table & calculate a nice % aligned
    try:
        self.general_stats_data[s_name]['total_reads'] = d['total_reads']
        self.general_stats_data[s_name]['mapped_reads'] = d['mapped_reads']
        d['percentage_aligned'] = (d['mapped_reads'] / d['total_reads'])*100
        self.general_stats_data[s_name]['percentage_aligned'] = d['percentage_aligned']
        self.general_stats_data[s_name]['general_error_rate'] = d['general_error_rate']*100
    except KeyError:
        pass

    # Save results
    if s_name in self.qualimap_bamqc_genome_results:
        log.debug("Duplicate genome results sample name found! Overwriting: {}".format(s_name))
    self.qualimap_bamqc_genome_results[s_name] = d
    self.add_data_source(f, s_name=s_name, section='genome_results')
989577
def _parse_qualimap_rnaseq(table):
    
    out = {}
    for row in table.find_all("tr"):
        col, val = [x.text for x in row.find_all("td")]
        col = col.replace(":", "").strip()
        val = val.replace(",", "")
        m = {col: val}
        if val.find("/") > -1:
            m = _parse_num_pct(col, val.replace("%", ""))
        out.update(m)
    return out
431769
def parse_tmhmm_long(tmhmm_results):
    
    with open(tmhmm_results) as f:
        lines = f.read().splitlines()

    infodict = defaultdict(dict)

    for l in lines:
        if 'Number of predicted TMHs:' in l:
            gene = l.split(' Number')[0].strip('# ')
            infodict[gene]['num_tm_helices'] = int(l.split(': ')[1])

        if 'WARNING' in l:
            log.warning('{}: no TMHMM predictions'.format(l))
            continue

        # TODO: POSSIBLE N-term signal sequence - parse this

        # Look for the lines without #, these are the TM predicted regions
        if '#' not in l:
            stuff = l.split()
            if stuff[1] == 'TMHMM2.0':

                gene = stuff[0]
                region = stuff[2]
                region_start = stuff[3]
                region_end = stuff[4]

                if 'sequence' in infodict[gene]:
                    tm_seq = infodict[gene]['sequence']
                else:
                    tm_seq = ''

                if region == 'outside':
                    info = 'O'
                elif region == 'inside':
                    info = 'I'
                elif region == 'TMhelix':
                    info = 'T'
                else:
                    log.error('{}: unknown region type'.format(info))
                    info = '-'

                for r in range(int(region_start), int(region_end) + 1):
                    tm_seq += info

                infodict[gene]['sequence'] = tm_seq

    return infodict
330209
def _parse_gene_coordinates(file):
        
        id_map = {}
        col = ['gene_curie', 'start', 'end', 'strand', 'build']
        if os.path.exists(os.path.join(os.path.dirname(__file__), file)):
            with open(os.path.join(os.path.dirname(__file__), file)) as tsvfile:
                reader = csv.reader(tsvfile, delimiter="\t")
                for row in reader:
                    id_map[row[col.index('gene_curie')]] = {
                        'start': row[col.index('start')],
                        'end': row[col.index('end')],
                        'strand': row[col.index('strand')],
                        'build': row[col.index('build')]
                    }
        return id_map
980598
def parse(self, file_obj, parsed_files=None):
        '''
        
        '''
        host = {'host': ['*'], 'config': {}}
        for line in file_obj:
            # Strip any leading or trailing whitespace from the line.
            # Refer to https://github.com/paramiko/paramiko/issues/499
            line = line.strip()
            if not line or line.startswith('#'):
                continue

            match = re.match(self.SETTINGS_REGEX, line)
            if not match:
                raise Exception('Unparsable line {}'.format(line))
            key = match.group(1).lower()
            value = match.group(2)

            if key == 'host':
                self._config.append(host)
                host = {
                    'host': self._get_hosts(value),
                    'config': {}
                }
            elif key == 'proxycommand' and value.lower() == 'none':
                # Store 'none' as None; prior to 3.x, it will get stripped out
                # at the end (for compatibility with issue #415). After 3.x, it
                # will simply not get stripped, leaving a nice explicit marker.
                host['config'][key] = None
            elif key == 'include':
                # support for Include directive in ssh_config
                path = value
                # the path can be relative to its parent configuration file
                if os.path.isabs(path) is False and path[0] != '~':
                    folder = os.path.dirname(file_obj.name)
                    path = os.path.join(folder, path)

                # expand the user home path
                path = os.path.expanduser(path)
                if parsed_files is None:
                    parsed_files = []

                # parse every included file
                for filename in glob.iglob(path):
                    if os.path.isfile(filename):
                        if filename in parsed_files:
                            raise Exception('Include loop detected in ssh config file: %s' % filename)
                        with open(filename) as fd:
                            parsed_files.append(filename)
                            self.parse(fd, parsed_files)

            else:
                if value.startswith('"') and value.endswith('"'):
                    value = value[1:-1]

                # identityfile, localforward, remoteforward keys are special
                # cases, since they are allowed to be specified multiple times
                # and they should be tried in order of specification.
                if key in ['identityfile', 'localforward', 'remoteforward']:
                    if key in host['config']:
                        host['config'][key].append(value)
                    else:
                        host['config'][key] = [value]
                elif key not in host['config']:
                    host['config'][key] = value
        self._config.append(host)
397691
def parse_quantitationesultsline(self, line):
        

        # [MS Quantitative Results]
        if line.startswith(self.QUANTITATIONRESULTS_KEY) \
                or line.startswith(self.QUANTITATIONRESULTS_NUMBEROFIDS) \
                or line.startswith(self.SIMILARITYSEARCHRESULTS_KEY) \
                or line.startswith(self.PEAK_TABLE_KEY):

            # Nothing to do, continue
            return 0

        # # of IDs \t23
        if line.startswith(self.QUANTITATIONRESULTS_HEADER_ID_NUMBER):
            self._quantitationresultsheader = [token.strip() for token
                                               in line.split('\t')
                                               if token.strip()]
            return 0

        # 1 \talpha-Pinene \tTarget \t0 \t93.00 \t7.738 \t7.680 \t7.795 \t2.480
        # \t344488 \t138926 \t0.02604 \tAuto \t2	\t7.812	\tLinear \t0 \t0
        # \t4.44061e+008	\t278569 \t0 \t0 \t38.94 \t38.58 \t0.00	\t98 \t92.00
        # \t0 \t0 \t38.94 \t38.58 \t91.00 \t0 \t0 \t38.93 \t40.02 \t0 \t0 \t0
        # \t0 \t0 \t0 \t0 #\t0 \t0 \t0 \t0 \t0 \t0 \t0 \t0 \t75.27 \tmg \t0.000
        splitted = [token.strip() for token in line.split('\t')]
        ar_id = self._header['Data File Name'].split('\\')[-1].split('.')[0]
        quantitation = {'DefaultResult': 'Conc.', 'AR': ar_id}
        for colname in self._quantitationresultsheader:
            quantitation[colname] = ''

        for i in range(len(splitted)):
            token = splitted[i]
            if i < len(self._quantitationresultsheader):
                colname = self._quantitationresultsheader[i]
                if colname in self.QUANTITATIONRESULTS_NUMERICHEADERS:
                    try:
                        quantitation[colname] = float(token)
                    except ValueError:
                        self.warn(
                            "No valid number ${token} in column "
                            "${index} (${column_name})",
                            mapping={"token": token,
                                     "index": str(i + 1),
                                     "column_name": colname},
                            numline=self._numline, line=line)
                        quantitation[colname] = token
                else:
                    quantitation[colname] = token

                # val = re.sub(r"\W", "", splitted[1])
                # self._addRawResult(quantitation['AR'],
                #                   values={val:quantitation},
                #                   override=False)
            elif token:
                self.err("Orphan value in column ${index} (${token})",
                         mapping={"index": str(i+1),
                                  "token": token},
                         numline=self._numline, line=line)

        result = quantitation[quantitation['DefaultResult']]
        column_name = quantitation['DefaultResult']
        result = self.zeroValueDefaultInstrumentResults(column_name,
                                                        result, line)
        quantitation[quantitation['DefaultResult']] = result

        val = re.sub(r"\W", "", splitted[1])
        self._addRawResult(quantitation['AR'],
                           values={val: quantitation},
                           override=False)
490749
def _parse_file_spec(self, spec):
        '''
        
        '''
        # separate wildcard specs into more specs
        if '*' in spec['file']:
            expanded_paths = _expand_paths(spec['file'])
            if not expanded_paths:
                return []
            expanded_specs = []
            for p in expanded_paths:
                _spec = copy.copy(spec)
                _spec['file'] = p
                expanded_specs.append(_spec)
            return expanded_specs

        else:
            return [spec]
141131
def parse (cls, line, lineno, log, cmddict=None):
    
    start  = line.find('{')
    stop   = line.find('}')
    pos    = SeqPos(line, lineno, start + 1, stop)
    result = cls(None, pos)

    if start >= 0 and stop >= start:
      attrs = { }
      pairs = line[start + 1:stop].split(',')

      for item in pairs:
        ncolons = item.count(':')
        if ncolons == 0:
          log.error('Missing colon in command attribute "%s".' % item, pos)
        elif ncolons > 1:
          log.error('Too many colons in command attribute "%s".' % item, pos)
        else:
          name, value = (s.strip() for s in item.split(':'))
          attrs[name] = value

      result = cls(attrs, pos)

    elif start != -1 or stop != -1:
      log.error('Incorrect command attribute curly brace placement.', pos)

    return result
1201147
def _parse_qualimap_globals(table):
    
    out = {}
    want = {"Mapped reads": _parse_num_pct,
            "Duplication rate": lambda k, v: {k: v}}
    for row in table.find_all("tr"):
        col, val = [x.text for x in row.find_all("td")]
        if col in want:
            out.update(want[col](col, val))
    return out
458316
def _get_biodata(base_file, args):
    
    with open(base_file) as in_handle:
        config = yaml.safe_load(in_handle)
    config["install_liftover"] = False
    config["genome_indexes"] = args.aligners
    ann_groups = config.pop("annotation_groups", {})
    config["genomes"] = [_setup_genome_annotations(g, args, ann_groups)
                         for g in config["genomes"] if g["dbkey"] in args.genomes]
    return config
1092272
def parsebam(self):
        
        # Threading is actually the worst - need multiprocessing to make this work at all
        logging.info('Parsing BAM files')
        # The sample objects are too big to get pickled. To hack our way around this, try to dump the sample object to
        # json, and have the processing function turn the object into a dictionary.
        json_files = list()
        with tempfile.TemporaryDirectory() as tmpdir:
            best_assemblies = list()
            sample_names = list()
            for sample in self.runmetadata:
                json_name = os.path.join(tmpdir, '{sn}.json'.format(sn=sample.name))
                best_assemblies.append(sample.general.bestassemblyfile)
                sample_names.append(sample.name)
                with open(json_name, 'w') as f:
                    json.dump(sample[self.analysistype].dump(), f, sort_keys=True, indent=4)
                json_files.append(json_name)
            p = multiprocessing.Pool(processes=self.cpus)
            analysis_type_list = [self.analysistype] * len(self.runmetadata)
            iupac_list = [self.iupac] * len(self.runmetadata)
            cutoff_list = [self.cutoff] * len(self.runmetadata)
            depth_list = [self.averagedepth] * len(self.runmetadata)
            allow_soft_clip_list = [self.allow_soft_clips] * len(self.runmetadata)
            sample_results = p.starmap(Sippr.parse_one_sample,
                                       zip(json_files, sample_names, best_assemblies, analysis_type_list,
                                           iupac_list, cutoff_list, depth_list, allow_soft_clip_list))
            p.close()
            p.join()
        # Since we had to json-ize the sample objects, we now need to update the metadata for everything.
        for sample in self.runmetadata:
                sample[self.analysistype].faidict = dict()
                sample[self.analysistype].results = dict()
                sample[self.analysistype].avgdepth = dict()
                sample[self.analysistype].resultssnp = dict()
                sample[self.analysistype].snplocations = dict()
                sample[self.analysistype].resultsgap = dict()
                sample[self.analysistype].gaplocations = dict()
                sample[self.analysistype].sequences = dict()
                sample[self.analysistype].maxcoverage = dict()
                sample[self.analysistype].mincoverage = dict()
                sample[self.analysistype].standarddev = dict()
                # Figure out which of the sample results to use.
                for sample_result in sample_results:
                    if sample_result['name'] == sample.name:
                        sample[self.analysistype].faidict = sample_result['faidict']
                        sample[self.analysistype].results = sample_result['results']
                        sample[self.analysistype].avgdepth = sample_result['avgdepth']
                        sample[self.analysistype].resultssnp = sample_result['resultssnp']
                        sample[self.analysistype].snplocations = sample_result['snplocations']
                        sample[self.analysistype].resultsgap = sample_result['resultsgap']
                        sample[self.analysistype].gaplocations = sample_result['gaplocations']
                        sample[self.analysistype].sequences = sample_result['sequences']
                        sample[self.analysistype].maxcoverage = sample_result['maxcoverage']
                        sample[self.analysistype].mincoverage = sample_result['mincoverage']
                        sample[self.analysistype].standarddev = sample_result['standarddev']
        logging.info('Done parsing BAM files')
799979
def parse_qc(self, qc_file):
        
        import pandas as pd
        series = pd.Series()
        try:
            with open(qc_file) as handle:
                line = handle.readlines()[0].strip().split("\t")  # list of strings per line
            series["NSC"] = line[-3]
            series["RSC"] = line[-2]
            series["qualityTag"] = line[-1]
        except:
            pass
        return series
1266308
def parse_qaml(self):
        
        logging.info('Parsing GenomeQAML outputs')
        # A dictionary to store the parsed excel file in a more readable format
        nesteddictionary = dict()
        # Use pandas to read in the CSV file, and convert the pandas data frame to a dictionary (.to_dict())
        dictionary = pandas.read_csv(self.qaml_report).to_dict()
        # Iterate through the dictionary - each header from the CSV file
        for header in dictionary:
            # Sample is the primary key, and value is the value of the cell for that primary key + header combination
            for sample, value in dictionary[header].items():
                # Update the dictionary with the new data
                try:
                    nesteddictionary[sample].update({header: value})
                # Create the nested dictionary if it hasn't been created yet
                except KeyError:
                    nesteddictionary[sample] = dict()
                    nesteddictionary[sample].update({header: value})
        # Get the results into the metadata object
        for sample in self.metadata:
            # Initialise the plasmid extractor genobject
            setattr(sample, self.analysistype, GenObject())
            # Initialise the list of all plasmids
            sample[self.analysistype].prediction = str()
            # Iterate through the dictionary of results
            for line in nesteddictionary:
                # Extract the sample name from the dictionary
                name = nesteddictionary[line]['Sample']
                # Ensure that the names match
                if name == sample.name:
                    # Append the plasmid name extracted from the dictionary to the list of plasmids
                    sample[self.analysistype].prediction = nesteddictionary[line]['Predicted_Class']
805933
def parse(self, filePath, skipLines=0, separator = ',', stringSeparator = '"', lineSeparator = '\n') :
		""""""
		
		self.filename = filePath
		f = open(filePath)
		if lineSeparator == '\n' :
			lines = f.readlines()
		else :
			lines = f.read().split(lineSeparator)
		f.flush()
		f.close()
		
		lines = lines[skipLines:]
		self.lines = []
		self.comments = []
		for l in lines :
			# print l
			if len(l) != 0 and l[0] != "#" :
				self.lines.append(l)
			elif l[0] == "#" :
				self.comments.append(l)

		self.separator = separator
		self.lineSeparator = lineSeparator
		self.stringSeparator = stringSeparator
		self.legend = collections.OrderedDict()
		
		i = 0
		for c in self.lines[0].lower().replace(stringSeparator, '').split(separator) :
			legendElement = c.strip()
			if legendElement not in self.legend :
				self.legend[legendElement] = i
			i+=1
	
		self.strLegend = self.lines[0].replace('\r', '\n').replace('\n', '')
		self.lines = self.lines[1:]
991154
def bam2es(
        bam_fn,
        es_fo,
        allowed_delta,
    ):
        

        es_fo.write("# RN:   read name" + os.linesep)
        es_fo.write("# Q:    is mapped with quality" + os.linesep)
        es_fo.write("# Chr:  chr id" + os.linesep)
        es_fo.write("# D:    direction" + os.linesep)
        es_fo.write("# L:    leftmost nucleotide" + os.linesep)
        es_fo.write("# R:    rightmost nucleotide" + os.linesep)
        es_fo.write("# Cat:  category of alignment assigned by LAVEnder" + os.linesep)
        es_fo.write("#         M_i    i-th segment is correctly mapped" + os.linesep)
        es_fo.write("#         m      segment should be unmapped but it is mapped" + os.linesep)
        es_fo.write("#         w      segment is mapped to a wrong location" + os.linesep)
        es_fo.write("#         U      segment is unmapped and should be unmapped" + os.linesep)
        es_fo.write("#         u      segment is unmapped and should be mapped" + os.linesep)
        es_fo.write("# Segs: number of segments" + os.linesep)
        es_fo.write("# " + os.linesep)
        es_fo.write("# RN\tQ\tChr\tD\tL\tR\tCat\tSegs" + os.linesep)

        with pysam.AlignmentFile(bam_fn, "rb") as sam:
            references_dict = {}

            for i in range(len(sam.references)):
                references_dict[sam.references[i]] = i + 1

            for read in sam:
                rnf_read_tuple = rnftools.rnfformat.ReadTuple()
                rnf_read_tuple.destringize(read.query_name)

                left = read.reference_start + 1
                right = read.reference_end
                chrom_id = references_dict[sam.references[read.reference_id]]

                nb_of_segments = len(rnf_read_tuple.segments)

                if rnf_read_tuple.segments[0].genome_id == 1:
                    should_be_mapped = True
                else:
                    should_be_mapped = False

                # read is unmapped
                if read.is_unmapped:
                    # read should be mapped
                    if should_be_mapped:
                        category = "u"
                    # read should be unmapped
                    else:
                        category = "U"
                # read is mapped
                else:
                    # read should be mapped
                    if should_be_mapped:
                        exists_corresponding_segment = False

                        for j in range(len(rnf_read_tuple.segments)):
                            segment = rnf_read_tuple.segments[j]
                            if (
                                (segment.left == 0 or abs(segment.left - left) <= allowed_delta)
                                and (segment.right == 0 or abs(segment.right - right) <= allowed_delta)
                                and (segment.left != 0 or segment.right == 0)
                                and (chrom_id == 0 or chrom_id == segment.chr_id)
                            ):
                                exists_corresponding_segment = True
                                segment = str(j + 1)
                                break

                        # read was mapped to correct location
                        if exists_corresponding_segment:  # exists ok location?
                            category = "M_" + segment
                        # read was mapped to incorrect location
                        else:
                            category = "w"
                    # read should be unmapped
                    else:
                        category = "m"

                es_fo.write(
                    "\t".join(
                        map(
                            str,
                            [
                                # read name
                                read.query_name,
                                # aligned?
                                "unmapped" if read.is_unmapped else "mapped_" + str(read.mapping_quality),
                                # reference id
                                chrom_id,
                                # direction
                                "R" if read.is_reverse else "F",
                                # left
                                left,
                                # right
                                right,
                                # assigned category
                                category,
                                # count of segments
                                nb_of_segments
                            ]
                        )
                    ) + os.linesep
                )
929846
def parse_ftype(input_file):
    
    if input_file.endswith(".bam"):
        return "bam"
    elif input_file.endswith(".fastq") or \
            input_file.endswith(".fq") or \
            input_file.endswith(".fq.gz") or \
            input_file.endswith(".fastq.gz"):
        return "fastq"
    else:
        raise TypeError("Type of input file ends in neither '.bam' "
                        "nor '.fastq' [file: '" + input_file + "']")
828339
def parse_psqs(psqs_results_file):
    

    # TODO: generalize column names for all results, save as dict instead

    psqs_results = pd.read_csv(psqs_results_file, sep='\t', header=None)
    psqs_results['pdb_file'] = psqs_results[0].apply(lambda x: str(x).strip('./').strip('.pdb'))
    psqs_results = psqs_results.rename(columns = {1:'psqs_local', 2:'psqs_burial', 3:'psqs_contact', 4:'psqs_total'}).drop(0, axis=1)
    psqs_results['u_pdb'] = psqs_results['pdb_file'].apply(lambda x: x.upper() if len(x)==4 else np.nan)
    psqs_results['i_entry_name'] = psqs_results['pdb_file'].apply(lambda x: x.split('_model1')[0] if len(x)>4 else np.nan)
    psqs_results = psqs_results[pd.notnull(psqs_results.psqs_total)]

    return psqs_results