Counts the number of reads for each UMI

    Expects formatted fastq files.
516969
def umi_histogram(fastq):
    ''' 
    '''
    annotations = detect_fastq_annotations(fastq)
    re_string = construct_transformed_regex(annotations)
    parser_re = re.compile(re_string)

    counter = collections.Counter()
    for read in read_fastq(fastq):
        match = parser_re.search(read).groupdict()
        counter[match['MB']] += 1

    for bc, count in counter.most_common():
        sys.stdout.write('{}\t{}\n'.format(bc, count))
24003
def read_examples(input_files, batch_size, shuffle, num_epochs=None):
  """"""
  files = []
  for e in input_files:
    for path in e.split(','):
      files.extend(file_io.get_matching_files(path))
  thread_count = multiprocessing.cpu_count()

  # The minimum number of instances in a queue from which examples are drawn
  # randomly. The larger this number, the more randomness at the expense of
  # higher memory requirements.
  min_after_dequeue = 1000

  # When batching data, the queue's capacity will be larger than the batch_size
  # by some factor. The recommended formula is (num_threads + a small safety
  # margin). For now, we use a single thread for reading, so this can be small.
  queue_size_multiplier = thread_count + 3

  # Convert num_epochs == 0 -> num_epochs is None, if necessary
  num_epochs = num_epochs or None

  # Build a queue of the filenames to be read.
  filename_queue = tf.train.string_input_producer(files, num_epochs, shuffle)

  options = tf.python_io.TFRecordOptions(
      compression_type=tf.python_io.TFRecordCompressionType.GZIP)
  example_id, encoded_example = tf.TFRecordReader(options=options).read_up_to(
      filename_queue, batch_size)

  if shuffle:
    capacity = min_after_dequeue + queue_size_multiplier * batch_size
    return tf.train.shuffle_batch(
        [example_id, encoded_example],
        batch_size,
        capacity,
        min_after_dequeue,
        enqueue_many=True,
        num_threads=thread_count)
  else:
    capacity = queue_size_multiplier * batch_size
    return tf.train.batch(
        [example_id, encoded_example],
        batch_size,
        capacity=capacity,
        enqueue_many=True,
        num_threads=thread_count)
1255154
def rsl_count_readings(self):
        """"""

        storage, output = self.sensor_log.count()
        return [Error.NO_ERROR, storage, output]
62368
def fastq(args):
    
    from jcvi.formats.fastq import FastqLite

    p = OptionParser(fastq.__doc__)
    p.add_option("--qv", type="int",
                 help="Use generic qv value [dafault: %default]")

    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    fastafile, = args
    fastqfile = fastafile.rsplit(".", 1)[0] + ".fastq"
    fastqhandle = open(fastqfile, "w")
    num_records = 0

    if opts.qv is not None:
        qv = chr(ord('!') + opts.qv)
        logging.debug("QV char '{0}' ({1})".format(qv, opts.qv))
    else:
        qv = None

    if qv:
        f = Fasta(fastafile, lazy=True)
        for name, rec in f.iteritems_ordered():
            r = FastqLite("@" + name, str(rec.seq).upper(), qv * len(rec.seq))
            print(r, file=fastqhandle)
            num_records += 1

    else:
        qualfile = get_qual(fastafile)
        for rec in iter_fasta_qual(fastafile, qualfile):
            SeqIO.write([rec], fastqhandle, "fastq")
            num_records += 1

    fastqhandle.close()
    logging.debug("A total of %d records written to `%s`" % \
            (num_records, fastqfile))
508546
def _add_umis_with_fastp(read_fq, umi_fq, out_fq, cores):
    
    with utils.open_gzipsafe(umi_fq) as in_handle:
        in_handle.readline()  # name
        umi_size = len(in_handle.readline().strip())
    cmd = ("fastp -Q -A -L -G -w 1 --in1 {read_fq} --in2 {umi_fq} "
           "--umi --umi_prefix UMI --umi_loc read2 --umi_len {umi_size} "
           "--out1 >(bgzip --threads {cores} -c > {out_fq}) --out2 /dev/null "
           "-j /dev/null -h /dev/null")
    do.run(cmd.format(**locals()), "Add UMIs to fastq file with fastp")
316245
def estimate_maximum_read_length(fastq_file, quality_format="fastq-sanger",
                                 nreads=1000):
    
    in_handle = SeqIO.parse(open_fastq(fastq_file), quality_format)
    lengths = []
    for _ in range(nreads):
        try:
            lengths.append(len(next(in_handle).seq))
        except StopIteration:
            break
    in_handle.close()
    return max(lengths)
25553
def count(args):
    
    p = OptionParser(count.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(p.print_help())

    frgfile, = args
    fp = open(frgfile)

    counts = defaultdict(int)
    for rec in iter_records(fp):
        counts[rec.type] += 1

    for type, cnt in sorted(counts.items()):
        print('{0}: {1}'.format(type, cnt), file=sys.stderr)
1053858
def count(self, seqs, nreport=100, scan_rc=True):
        
        for matches in self.scan(seqs, nreport, scan_rc):
            counts = [len(m) for m in matches]
            yield counts
463453
def read_fastq(filename):
    
    if not filename:
        return itertools.cycle((None,))
    if filename == "-":
        filename_fh = sys.stdin
    elif filename.endswith('gz'):
        if is_python3:
            filename_fh = gzip.open(filename, mode='rt')
        else:
            filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
    else:
        filename_fh = open(filename)
    return stream_fastq(filename_fh)
314623
def prep_fastq_inputs(in_files, data):
    
    if len(in_files) == 1 and _is_bam_input(in_files):
        out = _bgzip_from_bam(in_files[0], data["dirs"], data)
    elif len(in_files) == 1 and _is_cram_input(in_files):
        out = _bgzip_from_cram(in_files[0], data["dirs"], data)
    elif len(in_files) in [1, 2] and _ready_gzip_fastq(in_files, data):
        out = _symlink_in_files(in_files, data)
    else:
        if len(in_files) > 2:
            fpairs = fastq.combine_pairs(in_files)
            pair_types = set([len(xs) for xs in fpairs])
            assert len(pair_types) == 1
            fpairs.sort(key=lambda x: os.path.basename(x[0]))
            organized = [[xs[0] for xs in fpairs]]
            if len(fpairs[0]) > 1:
                organized.append([xs[1] for xs in fpairs])
            in_files = organized
        parallel = {"type": "local", "num_jobs": len(in_files),
                    "cores_per_job": max(1, data["config"]["algorithm"]["num_cores"] // len(in_files))}
        inputs = [{"in_file": x, "read_num": i, "dirs": data["dirs"], "config": data["config"],
                   "is_cwl": "cwl_keys" in data,
                   "rgnames": data["rgnames"]}
                  for i, x in enumerate(in_files) if x]
        out = run_multicore(_bgzip_from_fastq_parallel, [[d] for d in inputs], data["config"], parallel)
    return out
808248
def fastqRead(fileHandleOrFile):
    
    fileHandle = _getFileHandle(fileHandleOrFile)
    line = fileHandle.readline()
    while line != '':
        if line[0] == '@':
            name = line[1:-1]
            seq = fileHandle.readline()[:-1]
            plus = fileHandle.readline()
            if plus[0] != '+':
                raise RuntimeError("Got unexpected line: %s" % plus)
            qualValues = [ ord(i) for i in fileHandle.readline()[:-1] ]
            if len(seq) != len(qualValues):
                logger.critical("Got a mismatch between the number of sequence characters (%s) and number of qual values (%s) for sequence: %s, ignoring returning None" % (len(seq), len(qualValues), name))
                qualValues = None
            else:
                for i in qualValues:
                    if i < 33 or i > 126:
                        raise RuntimeError("Got a qual value out of range %s (range is 33 to 126)" % i)
            for i in seq:
                #For safety and sanity I only allows roman alphabet characters in fasta sequences.
                if not ((i >= 'A' and i <= 'Z') or (i >= 'a' and i <= 'z') or i == '-'):
                    raise RuntimeError("Invalid FASTQ character, ASCII code = \'%d\', found in input sequence %s" % (ord(i), name))
            yield name, seq, qualValues
        line = fileHandle.readline()
    if isinstance(fileHandleOrFile, "".__class__):
        fileHandle.close()
337360
def collect_read_counts(data, work_dir):
    
    out_file = os.path.join(work_dir, "%s-target-coverage.hdf5" % dd.get_sample_name(data))
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            params = ["-T", "CollectReadCounts", "-I", dd.get_align_bam(data),
                      "-L", tz.get_in(["regions", "bins", "target"], data),
                      "--interval-merging-rule", "OVERLAPPING_ONLY",
                      "-O", tx_out_file, "--format", "HDF5"]
            _run_with_memory_scaling(params, tx_out_file, data)
    return out_file
1372460
def ngram_counts(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    store.validate(corpus, catalogue)
    store.counts(catalogue, sys.stdout)
771506
def __read_single_fasta_query_lines(f):
    '''
    
    '''

    def readline():
        l = f.readline()
        if l == '':
            raise EOFError()
        return l

    rec = None
    try:
        l = readline()
        assert l.startswith('>')

        rec = [l]
        while True:
            pos = f.tell()
            l = readline()
            if l.startswith('>'):
                f.seek(pos, 0)
                break
            rec += [l]
    except EOFError:
        pass

    return rec
1236125
def count_duplicate_starts(bam_file, sample_size=10000000):
    
    count = Counter()
    with bam.open_samfile(bam_file) as samfile:
        # unmapped reads should not be counted
        filtered = ifilter(lambda x: not x.is_unmapped, samfile)
        def read_parser(read):
            return ":".join([str(read.tid), str(read.pos)])
        samples = utils.reservoir_sample(filtered, sample_size, read_parser)

    count.update(samples)
    return count
671836
public void process( String uniprotDatFileName,String fastaFileName ) throws Exception{

			FileReader fr = new FileReader(uniprotDatFileName);
			BufferedReader br = new BufferedReader(fr);
			String line = br.readLine();
			String id = "";
			StringBuffer sequence = new StringBuffer();
			ArrayList<ProteinSequence> seqCodingRegionsList = new ArrayList<ProteinSequence>();
			int count = 0;
			HashMap<String,String> uniqueGenes = new HashMap<String,String>();
			HashMap<String,String> uniqueSpecies = new HashMap<String,String>();
			while(line != null){
				if(line.startsWith("ID")){
					String[] data = line.split(" ");
					id = data[3];
				}else if(line.startsWith("SQ")){
					line = br.readLine();
					while(!line.startsWith("//")){

						for(int i = 0; i < line.length(); i++){
							char aa = line.charAt(i);
							if((aa >= 'A' && aa <= 'Z') || (aa >= 'a' && aa <= 'z' )){
								sequence.append(aa);
							}
						}
						line = br.readLine();
					}

				 //   System.out.println(">" + id);
				 //   System.out.println(sequence.toString());

					ProteinSequence seq = new ProteinSequence(sequence.toString() );
					seq.setAccession(new AccessionID(id));

					seqCodingRegionsList.add(seq);
					sequence = new StringBuffer();
					count++;
					if(count % 100 == 0)
						logger.info("Count: ", count);
					String[] parts = id.split("_");
					uniqueGenes.put(parts[0], "");
					uniqueSpecies.put(parts[1],"");
				}
				line = br.readLine();
			}
	   //     System.out.println("Unique Genes=" + uniqueGenes.size());
	   //     System.out.println("Unique Species=" + uniqueSpecies.size());
	   //     System.out.println("Total sequences=" + seqCodingRegionsList.size());
			FastaWriterHelper.writeProteinSequence(new File(fastaFileName), seqCodingRegionsList);

			br.close();
			fr.close();

	  //      System.out.println(uniqueGenes.keySet());
	  //      System.out.println("====================");
	  //      System.out.println(uniqueSpecies.keySet());


	}
847668
def process_files(count=None):
    

    pbar = enlighten.Counter(total=count, desc='Simple', unit='ticks',
                             bar_format=BAR_FMT, counter_format=COUNTER_FMT)

    for num in range(100):  # pylint: disable=unused-variable
        time.sleep(0.05)
        pbar.update(1.1)