Parse the Tophat alignment log file.
10602
def parse_tophat_log (self, raw_data):
        

        if 'Aligned pairs' in raw_data:
            # Paired end data
            regexes = {
                'overall_aligned_percent': r"([\d\.]+)% overall read mapping rate.",
                'concordant_aligned_percent': r"([\d\.]+)% concordant pair alignment rate.",
                'aligned_total': r"Aligned pairs:\s+(\d+)",
                'aligned_multimap': r"Aligned pairs:\s+\d+\n\s+of these:\s+(\d+)",
                'aligned_discordant': r"(\d+) \([\s\d\.]+%\) are discordant alignments",
                'total_reads': r"[Rr]eads:\n\s+Input\s*:\s+(\d+)",
            }
        else:
            # Single end data
            regexes = {
                'total_reads': r"[Rr]eads:\n\s+Input\s*:\s+(\d+)",
                'aligned_total': r"Mapped\s*:\s+(\d+)",
                'aligned_multimap': r"of these\s*:\s+(\d+)",
                'overall_aligned_percent': r"([\d\.]+)% overall read mapping rate.",
            }

        parsed_data = {}
        for k, r in regexes.items():
            r_search = re.search(r, raw_data, re.MULTILINE)
            if r_search:
                parsed_data[k] = float(r_search.group(1))
        if len(parsed_data) == 0: return None
        parsed_data['concordant_aligned_percent'] = parsed_data.get('concordant_aligned_percent', 0)
        parsed_data['aligned_total'] = parsed_data.get('aligned_total', 0)
        parsed_data['aligned_multimap'] = parsed_data.get('aligned_multimap', 0)
        parsed_data['aligned_discordant'] = parsed_data.get('aligned_discordant', 0)
        parsed_data['unaligned_total'] = parsed_data['total_reads'] - parsed_data['aligned_total']
        parsed_data['aligned_not_multimapped_discordant'] = parsed_data['aligned_total'] - parsed_data['aligned_multimap'] - parsed_data['aligned_discordant']
        return parsed_data
375341
def parse_cutadapt_logs(self, f):
        
        fh = f['f']
        regexes = {
            '1.7': {
                'bp_processed': "Total basepairs processed:\s*([\d,]+) bp",
                'bp_written': "Total written \(filtered\):\s*([\d,]+) bp",
                'quality_trimmed': "Quality-trimmed:\s*([\d,]+) bp",
                'r_processed': "Total reads processed:\s*([\d,]+)",
                'r_with_adapters': "Reads with adapters:\s*([\d,]+)"
            },
            '1.6': {
                'r_processed': "Processed reads:\s*([\d,]+)",
                'bp_processed': "Processed bases:\s*([\d,]+) bp",
                'r_trimmed': "Trimmed reads:\s*([\d,]+)",
                'quality_trimmed': "Quality-trimmed:\s*([\d,]+) bp",
                'bp_trimmed': "Trimmed bases:\s*([\d,]+) bp",
                'too_short': "Too short reads:\s*([\d,]+)",
                'too_long': "Too long reads:\s*([\d,]+)",
            }
        }
        s_name = None
        cutadapt_version = '1.7'
        log_section = None
        for l in fh:
            # New log starting
            if 'cutadapt' in l:
                s_name = None
                c_version = re.match(r'This is cutadapt ([\d\.]+)', l)
                if c_version:
                    try:
                        assert(StrictVersion(c_version.group(1)) <= StrictVersion('1.6'))
                        cutadapt_version = '1.6'
                    except:
                        cutadapt_version = '1.7'
                c_version_old = re.match(r'cutadapt version ([\d\.]+)', l)
                if c_version_old:
                    try:
                        assert(StrictVersion(c_version.group(1)) <= StrictVersion('1.6'))
                        cutadapt_version = '1.6'
                    except:
                        # I think the pattern "cutadapt version XX" is only pre-1.6?
                        cutadapt_version = '1.6'
            # Get sample name from end of command line params
            if l.startswith('Command line parameters'):
                s_name = l.split()[-1]
                # Manage case where sample name is '-' (reading from stdin)
                if s_name == '-':
                    s_name = f['s_name']
                else:
                    s_name = self.clean_s_name(s_name, f['root'])
                if s_name in self.cutadapt_data:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
                self.cutadapt_data[s_name] = dict()

            if s_name is not None:
                self.add_data_source(f, s_name)

                # Search regexes for overview stats
                for k, r in regexes[cutadapt_version].items():
                    match = re.search(r, l)
                    if match:
                        self.cutadapt_data[s_name][k] = int(match.group(1).replace(',', ''))

                # Starting a new section
                if '===' in l:
                    log_section = l.strip().strip('=').strip()

                # Histogram showing lengths trimmed
                if 'length' in l and 'count' in l and 'expect' in l:
                    plot_sname = s_name
                    if log_section is not None:
                        plot_sname = '{} - {}'.format(s_name, log_section)
                    self.cutadapt_length_counts[plot_sname] = dict()
                    self.cutadapt_length_exp[plot_sname] = dict()
                    self.cutadapt_length_obsexp[plot_sname] = dict()
                    # Nested loop to read this section while the regex matches
                    for l in fh:
                        r_seqs = re.search("^(\d+)\s+(\d+)\s+([\d\.]+)", l)
                        if r_seqs:
                            a_len = int(r_seqs.group(1))
                            self.cutadapt_length_counts[plot_sname][a_len] = int(r_seqs.group(2))
                            self.cutadapt_length_exp[plot_sname][a_len] = float(r_seqs.group(3))
                            if float(r_seqs.group(3)) > 0:
                                self.cutadapt_length_obsexp[plot_sname][a_len] = float(r_seqs.group(2)) / float(r_seqs.group(3))
                            else:
                                # Cheating, I know. Infinity is difficult to plot.
                                self.cutadapt_length_obsexp[plot_sname][a_len] = float(r_seqs.group(2))
                        else:
                            break

        # Calculate a few extra numbers of our own
        for s_name, d in self.cutadapt_data.items():
            if 'bp_processed' in d and 'bp_written' in d:
                self.cutadapt_data[s_name]['percent_trimmed'] = (float(d['bp_processed'] - d['bp_written']) / d['bp_processed']) * 100
            elif 'bp_processed' in d and 'bp_trimmed' in d:
                self.cutadapt_data[s_name]['percent_trimmed'] = ((float(d.get('bp_trimmed', 0)) + float(d.get('quality_trimmed', 0))) / d['bp_processed']) * 100
400168
def _parse_wikiheadlines(path):
  """"""
  lang_match = re.match(r".*\.([a-z][a-z])-([a-z][a-z])$", path)
  assert lang_match is not None, "Invalid Wikiheadlines filename: %s" % path
  l1, l2 = lang_match.groups()
  with tf.io.gfile.GFile(path) as f:
    for line in f:
      s1, s2 = line.split("|||")
      yield {
          l1: s1.strip(),
          l2: s2.strip()
      }
1373113
def split_log(logf):
        """"""
        flashpatt = re.compile(
            r'\[FLASH\] Fast Length Adjustment of SHort reads\n(.+?)\[FLASH\] FLASH', flags=re.DOTALL)
        return flashpatt.findall(logf)
1263848
def parse_log(file_path):
    
    if not os.path.isfile(file_path):
        return elements.error("Output Log", "Could not open file: " + file_path.split(os.sep)[-1])

    headers = ["Converged Iterations",
               "Avg. Iterations to Converge",
               "Processor Count",
               "Dycore Type"]

    with open(file_path, 'r') as f:
        dycore_types = {"0": "Glide",
                        "1": "Glam",
                        "2": "Glissade",
                        "3": "Albany_felix",
                        "4": "BISICLES"}
        curr_step = 0
        proc_count = 0
        iter_number = 0
        converged_iters = []
        iters_to_converge = []
        for line in f:
            split = line.split()
            if ('CISM dycore type' in line):
                if line.split()[-1] == '=':
                    dycore_type = dycore_types[next(f).strip()]
                else:
                    dycore_type = dycore_types[line.split()[-1]]
            elif ('total procs' in line):
                proc_count += int(line.split()[-1])
            elif ('Nonlinear Solver Step' in line):
                curr_step = int(line.split()[4])
            elif ('Compute ice velocities, time = ' in line):
                converged_iters.append(curr_step)
                curr_step = float(line.split()[-1])
            elif ('"SOLVE_STATUS_CONVERGED"' in line):
                split = line.split()
                iters_to_converge.append(int(split[split.index('"SOLVE_STATUS_CONVERGED"') + 2]))
            elif ("Compute dH/dt" in line):
                iters_to_converge.append(int(iter_number))
            elif len(split) > 0 and split[0].isdigit():
                iter_number = split[0]
        if iters_to_converge == []:
            iters_to_converge.append(int(iter_number))
    data = {
        "Dycore Type": dycore_type,
        "Processor Count": proc_count,
        "Converged Iterations": len(converged_iters),
        "Avg. Iterations to Converge": np.mean(iters_to_converge)
    }
    return elements.table("Output Log", headers, data)
1206765
def parse_line(self, line, lineno):
        """"""
        # TaskCluster logs are a bit wonky.
        #
        # TaskCluster logs begin with output coming from TaskCluster itself,
        # before it has transitioned control of the task to the configured
        # process. These "internal" logs look like the following:
        #
        #   [taskcluster 2016-09-09 17:41:43.544Z] Worker Group: us-west-2b
        #
        # If an error occurs during this "setup" phase, TaskCluster may emit
        # lines beginning with ``[taskcluster:error]``.
        #
        # Once control has transitioned from TaskCluster to the configured
        # task process, lines can be whatever the configured process emits.
        # The popular ``run-task`` wrapper prefixes output to emulate
        # TaskCluster's "internal" logs. e.g.
        #
        #   [vcs 2016-09-09T17:45:02.842230Z] adding changesets
        #
        # This prefixing can confuse error parsing. So, we strip it.
        #
        # Because regular expression matching and string manipulation can be
        # expensive when performed on every line, we only strip the TaskCluster
        # log prefix if we know we're in a TaskCluster log.

        # First line of TaskCluster logs almost certainly has this.
        if line.startswith('[taskcluster '):
            self.is_taskcluster = True

        # For performance reasons, only do this if we have identified as
        # a TC task.
        if self.is_taskcluster:
            line = re.sub(self.RE_TASKCLUSTER_NORMAL_PREFIX, "", line)

        if self.is_error_line(line):
            self.add(line, lineno)
1051763
def parsed_file(config_file):
    """"""
    parser = ConfigParser(allow_no_value=True)
    parser.readfp(config_file)
    return parser
1026369
private int getLogFileIndex(String fileName) {
		int fileIndex = 0;
		try {
			fileIndex = Integer.parseInt(fileName.substring(fileName
					.lastIndexOf(COUNT_SEPARATOR) + 1));
		} catch (NumberFormatException e) {
			e.printStackTrace();
		}
		return fileIndex;
	}
603939
def tophat(args):
    
    from jcvi.apps.bowtie import check_index
    from jcvi.formats.fastq import guessoffset

    p = OptionParser(tophat.__doc__)
    p.add_option("--gtf", help="Reference annotation [default: %default]")
    p.add_option("--single", default=False, action="store_true",
                 help="Single end mapping")
    p.add_option("--intron", default=15000, type="int",
                 help="Max intron size [default: %default]")
    p.add_option("--dist", default=-50, type="int",
                 help="Mate inner distance [default: %default]")
    p.add_option("--stdev", default=50, type="int",
                 help="Mate standard deviation [default: %default]")
    p.set_phred()
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    num = 1 if opts.single else 2
    folder, reference = args
    reference = check_index(reference)
    for p, prefix in iter_project(folder, n=num):
        outdir = "{0}_tophat".format(prefix)
        outfile = op.join(outdir, "accepted_hits.bam")
        if op.exists(outfile):
            logging.debug("File `{0}` found. Skipping.".format(outfile))
            continue

        cmd = "tophat -p {0}".format(opts.cpus)
        if opts.gtf:
            cmd += " -G {0}".format(opts.gtf)
        cmd += " -o {0}".format(outdir)

        if num == 1:  # Single-end
            a, = p
        else:  # Paired-end
            a, b = p
            cmd += " --max-intron-length {0}".format(opts.intron)
            cmd += " --mate-inner-dist {0}".format(opts.dist)
            cmd += " --mate-std-dev {0}".format(opts.stdev)

        phred = opts.phred or str(guessoffset([a]))
        if phred == "64":
            cmd += " --phred64-quals"
        cmd += " {0} {1}".format(reference, " ".join(p))

        sh(cmd)
671486
void parseOneFile(final Path inputPath, final File outputFolder) throws IOException {
    try (final TFile.Reader.Scanner scanner = this.getScanner(inputPath)) {
      while (!scanner.atEnd()) {
        new LogFileEntry(scanner.entry()).write(outputFolder);
        scanner.advance();
      }
    }
  }
941009
def parse(self):
        """"""
        start = time.time()

        if self.cache:
            self.load_cache()

        if not self.cache_loaded:
            super(ObjParser, self).parse()

        logger.info("%s: Load time: %s", self.file_name, time.time() - start)
680424
private String[] parseArgLine( final String toProcess )
    {
        if ( toProcess == null || toProcess.length() == 0 )
        {
            // no command? no string
            return new String[0];
        }

        // parse with a simple finite state machine

        final int normal = 0;
        final int inQuote = 1;
        final int inDoubleQuote = 2;
        int state = normal;
        final StringTokenizer tok = new StringTokenizer( toProcess, "\"\' ", true );
        final ArrayList<String> list = new ArrayList<String>();
        StringBuilder current = new StringBuilder();
        boolean lastTokenHasBeenQuoted = false;

        while ( tok.hasMoreTokens() )
        {
            final String nextTok = tok.nextToken();
            switch ( state )
            {
                case inQuote:
                    if ( "\'".equals( nextTok ) )
                    {
                        lastTokenHasBeenQuoted = true;
                        state = normal;
                    }
                    else
                    {
                        current.append( nextTok );
                    }
                    break;
                case inDoubleQuote:
                    if ( "\"".equals( nextTok ) )
                    {
                        lastTokenHasBeenQuoted = true;
                        state = normal;
                    }
                    else
                    {
                        current.append( nextTok );
                    }
                    break;
                default:
                    if ( "\'".equals( nextTok ) )
                    {
                        state = inQuote;
                    }
                    else if ( "\"".equals( nextTok ) )
                    {
                        state = inDoubleQuote;
                    }
                    else if ( " ".equals( nextTok ) )
                    {
                        if ( lastTokenHasBeenQuoted || current.length() != 0 )
                        {
                            list.add( current.toString() );
                            current = new StringBuilder();
                        }
                    }
                    else
                    {
                        current.append( nextTok );
                    }
                    lastTokenHasBeenQuoted = false;
                    break;
            }
        }

        if ( lastTokenHasBeenQuoted || current.length() != 0 )
        {
            list.add( current.toString() );
        }

        if ( state == inQuote || state == inDoubleQuote )
        {
            throw new IllegalArgumentException( "Unbalanced quotes in " + toProcess );
        }

        final String[] args = new String[list.size()];
        return list.toArray( args );
    }
154452
def parse_log_line(line):
    
    matches = re.search(r'^\[([^\]]+)\] ([^:]+: .*)', line)
    error = re.search(r'Traceback', line)
    if error:
        return {'line': line, 'step': 'error'}
    if not matches:
        return {'line': line, 'step': None}

    tstamp = matches.group(1)
    msg = matches.group(2)

    if not msg.find('Timing: ') >= 0:
        return {'line': line, 'step': None}

    when = datetime.strptime(tstamp, '%Y-%m-%dT%H:%MZ').replace(
        tzinfo=pytz.timezone('UTC'))
    step = msg.split(":")[-1].strip()
    return {'line': line, 'step': step, 'when': when}
1258175
def parse_reports(self):
    

    # Set up vars
    self.picard_alignment_metrics = dict()

    # Go through logs and find Metrics
    for f in self.find_log_files('picard/alignment_metrics', filehandles=True):
        parsed_data = dict()
        s_name = None
        keys = None
        for l in f['f']:
            # New log starting
            if 'AlignmentSummaryMetrics' in l and 'INPUT' in l:
                s_name = None
                keys = None
                # Pull sample name from input
                fn_search = re.search(r"INPUT(?:=|\s+)(\[?[^\s]+\]?)", l, flags=re.IGNORECASE)
                if fn_search:
                    s_name = os.path.basename(fn_search.group(1).strip('[]'))
                    s_name = self.clean_s_name(s_name, f['root'])
                    parsed_data[s_name] = dict()

            if s_name is not None:
                if 'AlignmentSummaryMetrics' in l and '## METRICS CLASS' in l:
                    keys = f['f'].readline().strip("\n").split("\t")
                elif keys:
                    vals = l.strip("\n").split("\t")
                    if len(vals) == len(keys):
                        # Ignore the FIRST_OF_PAIR / SECOND_OF_PAIR data to simplify things
                        if vals[0] == 'PAIR' or vals[0] == 'UNPAIRED':
                            for i, k in enumerate(keys):
                                try:
                                    parsed_data[s_name][k] = float(vals[i])
                                except ValueError:
                                    parsed_data[s_name][k] = vals[i]
                    else:
                        s_name = None
                        keys = None

        # Remove empty dictionaries
        for s_name in list(parsed_data.keys()):
            if len(parsed_data[s_name]) == 0:
                parsed_data.pop(s_name, None)

        # Manipulate sample names if multiple baits found
        for s_name in parsed_data.keys():
            if s_name in self.picard_alignment_metrics:
                log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
            self.add_data_source(f, s_name, section='AlignmentSummaryMetrics')
            self.picard_alignment_metrics[s_name] = parsed_data[s_name]

    # Filter to strip out ignored sample names
    self.picard_alignment_metrics = self.ignore_samples(self.picard_alignment_metrics)

    if len(self.picard_alignment_metrics) > 0:

        # Write parsed data to a file
        self.write_data_file(self.picard_alignment_metrics, 'multiqc_picard_AlignmentSummaryMetrics')

        # Add to general stats table
        self.general_stats_headers['PCT_PF_READS_ALIGNED'] = {
            'title': '% Aligned',
            'description': 'Percent of aligned reads',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'format': '{:,.0f}',
            'scale': 'RdYlGn',
            'modify': lambda x: self.multiply_hundred(x)
        }
        for s_name in self.picard_alignment_metrics:
            if s_name not in self.general_stats_data:
                self.general_stats_data[s_name] = dict()
            self.general_stats_data[s_name].update( self.picard_alignment_metrics[s_name] )



        # Make the bar plot of alignment read count
        pdata = dict()
        for s_name in self.picard_alignment_metrics.keys():
            pdata[s_name] = dict()
            # Picard reports both reads for PE data. Divide it by two as most people will expect # clusters
            if self.picard_alignment_metrics[s_name]['CATEGORY'] == 'PAIR':
                pdata[s_name]['total_reads'] = self.picard_alignment_metrics[s_name]['TOTAL_READS'] / 2
                pdata[s_name]['aligned_reads'] = self.picard_alignment_metrics[s_name]['PF_READS_ALIGNED'] / 2
            else:
                pdata[s_name]['total_reads'] = self.picard_alignment_metrics[s_name]['TOTAL_READS']
                pdata[s_name]['aligned_reads'] = self.picard_alignment_metrics[s_name]['PF_READS_ALIGNED']
            pdata[s_name]['unaligned_reads'] = pdata[s_name]['total_reads'] - pdata[s_name]['aligned_reads']

        keys = OrderedDict()
        keys['aligned_reads'] = {'name': 'Aligned Reads'}
        keys['unaligned_reads'] = {'name': 'Unaligned Reads'}

        # Config for the plot
        pconfig = {
            'id': 'picard_aligned_reads',
            'title': 'Picard: Aligned Reads',
            'ylab': '# Reads',
            'cpswitch_counts_label': 'Number of Reads',
        }

        self.add_section (
            name = 'Alignment Summary',
            anchor = 'picard-alignmentsummary',
            description = "Plase note that Picard's read counts are divided by two for paired-end data.",
            plot = bargraph.plot(pdata, keys, pconfig)
        )

    # Return the number of detected samples to the parent module
    return len(self.picard_alignment_metrics)
440153
def parse_hisat2_logs(self, f):
        

        # Regexes
        regexes = {
            'unpaired_total': r"Total(?: unpaired)? reads: (\d+)",
            'unpaired_aligned_none': r"Aligned 0 times?: (\d+) \([\d\.]+%\)",
            'unpaired_aligned_one': r"Aligned 1 time: (\d+) \([\d\.]+%\)",
            'unpaired_aligned_multi': r"Aligned >1 times: (\d+) \([\d\.]+%\)",
    		'paired_total': r"Total pairs: (\d+)",
    		'paired_aligned_none': r"Aligned concordantly or discordantly 0 time: (\d+) \([\d\.]+%\)",
    		'paired_aligned_one': r"Aligned concordantly 1 time: (\d+) \([\d\.]+%\)",
    		'paired_aligned_multi': r"Aligned concordantly >1 times: (\d+) \([\d\.]+%\)",
    		'paired_aligned_discord_one': r"Aligned discordantly 1 time: (\d+) \([\d\.]+%\)",
        }

        # Go through log file line by line
        s_name = f['s_name']
        parsed_data = {}

        for l in f['f']:

            # Attempt in vain to find original hisat2 command, logged by another program
            hscmd = re.search(r"hisat2 .+ -[1U] ([^\s,]+)", l)
            if hscmd:
                s_name = self.clean_s_name(hscmd.group(1), f['root'])
                log.debug("Found a HISAT2 command, updating sample name to '{}'".format(s_name))

            # Run through all regexes
            for k, r in regexes.items():
                match = re.search(r, l)
                if match:
                    parsed_data[k] = int(match.group(1))

            # Overall alignment rate
            overall = re.search(r"Overall alignment rate: ([\d\.]+)%", l)
            if overall:
                parsed_data['overall_alignment_rate'] = float(overall.group(1))

                # Save parsed data
                if s_name in self.hisat2_data:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
                self.add_data_source(f, s_name)
                self.hisat2_data[s_name] = parsed_data

                # Reset in case we find more in this log file
                s_name = f['s_name']
                parsed_data = {}
1301678
def _parse(self):
        """"""
        if not self.pattern:
            raise TiffSequence.ParseError('invalid pattern')
        pattern = re.compile(self.pattern, re.IGNORECASE | re.VERBOSE)
        matches = pattern.findall(os.path.split(self.files[0])[-1])
        if not matches:
            raise TiffSequence.ParseError('pattern does not match file names')
        matches = matches[-1]
        if len(matches) % 2:
            raise TiffSequence.ParseError(
                'pattern does not match axis name and index')
        axes = ''.join(m for m in matches[::2] if m)
        if not axes:
            raise TiffSequence.ParseError('pattern does not match file names')

        indices = []
        for fname in self.files:
            fname = os.path.split(fname)[-1]
            matches = pattern.findall(fname)[-1]
            if axes != ''.join(m for m in matches[::2] if m):
                raise ValueError('axes do not match within image sequence')
            indices.append([int(m) for m in matches[1::2] if m])
        shape = tuple(numpy.max(indices, axis=0))
        startindex = tuple(numpy.min(indices, axis=0))
        shape = tuple(i-j+1 for i, j in zip(shape, startindex))
        if product(shape) != len(self.files):
            log.warning(
                'TiffSequence: files are missing. Missing data are zeroed')

        self.axes = axes.upper()
        self.shape = shape
        self._indices = indices
        self._startindex = startindex
901825
def parse_bbt(self, fh):
        
        parsed_data = OrderedDict()
        headers = None
        for l in fh:
            s = l.split("\t")
            if headers is None:
                headers = s
            else:
                parsed_data[s[0]] = dict()
                for i, h in enumerate(headers[1:]):
                    parsed_data[s[0]][h] = float(s[i+1])

        return parsed_data