Provide either a file with one cutoff per motif or a single cutoff
        returns a hash with motif id as key and cutoff as value
9791
def parse_cutoff(motifs, cutoff, default=0.9):
    
    
    cutoffs = {}
    if os.path.isfile(str(cutoff)):
        for i,line in enumerate(open(cutoff)):
            if line != "Motif\tScore\tCutoff\n":
                try:
                    motif,_,c = line.strip().split("\t")
                    c = float(c)
                    cutoffs[motif] = c
                except Exception as e:
                    sys.stderr.write("Error parsing cutoff file, line {0}: {1}\n".format(e, i + 1))
                    sys.exit(1)
    else:
        for motif in motifs:
            cutoffs[motif.id] = float(cutoff)
    
    for motif in motifs:
        if not motif.id in cutoffs:
            sys.stderr.write("No cutoff found for {0}, using default {1}\n".format(motif.id, default))
            cutoffs[motif.id] = default
    return cutoffs
79567
def hash_file(file_path, block_size = 65536):
    
    sha = hashlib.sha256()
    with open(file_path, 'rb') as h_file:
        file_buffer = h_file.read(block_size)
        while len(file_buffer) > 0:
            sha.update(file_buffer)
            file_buffer = h_file.read(block_size)
    return sha.hexdigest()
1181054
def cutoff_w_expression(vcf_file, expression, data, name="+", filterext="",
                      extra_cmd="", limit_regions="variant_regions"):
    
    base, ext = utils.splitext_plus(vcf_file)
    out_file = "{base}-filter{filterext}{ext}".format(**locals())
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            if vcfutils.vcf_has_variants(vcf_file):
                bcftools = config_utils.get_program("bcftools", data["config"])
                bgzip_cmd = "| bgzip -c" if out_file.endswith(".gz") else ""
                intervals = ""
                if limit_regions == "variant_regions":
                    variant_regions = dd.get_variant_regions(data)
                    if variant_regions:
                        intervals = "-T %s" % vcfutils.bgzip_and_index(variant_regions, data["config"])
                cmd = ("{bcftools} filter -O v {intervals} --soft-filter '{name}' "
                       "-e '{expression}' -m '+' {vcf_file} {extra_cmd} {bgzip_cmd} > {tx_out_file}")
                do.run(cmd.format(**locals()),
                       "Cutoff-based soft filtering %s with %s" % (vcf_file, expression), data)
            else:
                shutil.copy(vcf_file, out_file)
    if out_file.endswith(".vcf.gz"):
        out_file = vcfutils.bgzip_and_index(out_file, data["config"])
    return out_file
429781
def parse_align_file(file_in):
    
    loc_id = 1
    bedfile_clusters = ""
    bamfile = pybedtools.BedTool(file_in)
    bed = pybedtools.BedTool.bam_to_bed(bamfile)
    for c, start, end, name, q, strand in bed:
        loc_id += 1
        bedfile_clusters += "%s\t%s\t%s\t%s\t%s\t%s\n" % \
                            (c, start, end, name, loc_id, strand)
    return bedfile_clusters
90391
def _parse_openssh_output(lines, fingerprint_hash_type=None):
    '''
    
    '''
    for line in lines:
        # We don't need any whitespace-only containing lines or arbitrary doubled newlines
        line = line.strip()
        if line == '':
            continue
        line += '\n'

        if line.startswith('#'):
            continue
        try:
            hostname, enc, key = line.split()
        except ValueError:  # incorrect format
            continue
        fingerprint = _fingerprint(key,
                                   fingerprint_hash_type=fingerprint_hash_type)
        if not fingerprint:
            continue
        yield {'hostname': hostname, 'key': key, 'enc': enc,
               'fingerprint': fingerprint}
78380
def sample(self, hash, limit=None, offset=None):
        ''''''

        uri = self._uris['sample'].format(hash)
        params = {'limit': limit, 'offset': offset}

        return self.get_parse(uri, params)
624999
def as_dict( self, key="id" ):
        
        rval = {}
        for motif in self:
            rval[ getattr( motif, key ) ] = motif
        return rval
547964
def hashes_get(versions_file, base_path):
    
    files = versions_file.files_get_all()
    result = {}
    for f in files:
        try:
            result[f] = functions.md5_file(base_path + f)
        except IOError:
            # Not all files exist for all versions.
            pass

    return result
243147
def _one_to_one(self):
        
        if not self.file_generator.to_path_compression:
            self._to_textfile(self.file_generator)
        elif self.file_generator.to_path_compression == "gz":
            self._to_gzipfile(self.file_generator)
        elif self.file_generator.to_path_compression == "bz2":
            self._to_bz2file(self.file_generator)
        elif self.file_generator.to_path_compression in ("tar", "tar.gz", "tar.bz2", "zip"):
            raise TypeError('One-to-many conversion, cannot convert "{}" into "{}"'.format(self.file_generator.from_path,
                                                                                           self.file_generator.to_path))
        else:
            raise TypeError('Unknown format: "{}"'.format(self.file_generator.to_path))
263857
def pp_predict_motifs(fastafile, outfile, analysis="small", organism="hg18", single=False, background="", tools=None, job_server=None, ncpus=8, max_time=-1, stats_fg=None, stats_bg=None):
    
    if tools is None:
        tools = {}

    config = MotifConfig()

    if not tools:
        tools = dict([(x,1) for x in config.get_default_params["tools"].split(",")])
    
    #logger = logging.getLogger('gimme.prediction.pp_predict_motifs')

    wmin = 5 
    step = 1
    if analysis in ["large","xl"]:
        step = 2
        wmin = 6
    
    analysis_max = {"xs":5,"small":8, "medium":10,"large":14, "xl":20}
    wmax = analysis_max[analysis]

    if analysis == "xs":
        sys.stderr.write("Setting analysis xs to small")
        analysis = "small"

    
    if not job_server:
        n_cpus = int(config.get_default_params()["ncpus"])
        job_server = Pool(processes=n_cpus, maxtasksperchild=1000) 
    
    jobs = {}
    
    result = PredictionResult(
                outfile, 
                fg_file=stats_fg, 
                background=stats_bg,
                job_server=job_server,
                )
    
    # Dynamically load all tools
    toolio = [x[1]() for x in inspect.getmembers(
                                                tool_classes, 
                                                lambda x: 
                                                        inspect.isclass(x) and 
                                                        issubclass(x, tool_classes.MotifProgram)
                                                ) if x[0] != 'MotifProgram']
    
    # TODO:
    # Add warnings for running time: Weeder, GADEM
        
    ### Add all jobs to the job_server ###
    params = {
            'analysis': analysis, 
            'background':background, 
            "single":single, 
            "organism":organism
            }
    
    # Tools that don't use a specified width usually take longer
    # ie. GADEM, XXmotif, MEME
    # Start these first.
    for t in [tool for tool in toolio if not tool.use_width]:
        if t.name in tools and tools[t.name]:
            logger.debug("Starting %s job", t.name)
            job_name = t.name
            jobs[job_name] = job_server.apply_async(
                        _run_tool,
                        (job_name, t, fastafile, params), 
                        callback=result.add_motifs)
        else:
            logger.debug("Skipping %s", t.name)

    for t in [tool for tool in toolio if tool.use_width]:
        if t.name in tools and tools[t.name]:
            for i in range(wmin, wmax + 1, step):
                logger.debug("Starting %s job, width %s", t.name, i)
                job_name = "%s_width_%s" % (t.name, i)
                my_params = params.copy()
                my_params['width'] = i
                jobs[job_name] = job_server.apply_async(
                    _run_tool,
                    (job_name, t, fastafile, my_params), 
                    callback=result.add_motifs)
        else:
            logger.debug("Skipping %s", t.name)
    
    logger.info("all jobs submitted")
    for job in jobs.values():
        job.get()

    result.wait_for_stats()
    ### Wait until all jobs are finished or the time runs out ###
#    start_time = time()    
#    try:
#        # Run until all jobs are finished
#        while len(result.finished) < len(jobs.keys()) and (not(max_time) or time() - start_time < max_time):
#            pass
#        if len(result.finished) < len(jobs.keys()):
#            logger.info("Maximum allowed running time reached, destroying remaining jobs")
#            job_server.terminate()
#            result.submit_remaining_stats()
#    ### Or the user gets impatient... ###
#    except KeyboardInterrupt:
#        # Destroy all running jobs
#        logger.info("Caught interrupt, destroying all running jobs")
#        job_server.terminate()
#        result.submit_remaining_stats()
#        
#    
#    if stats_fg and stats_bg:
#        logger.info("waiting for motif statistics")
#        n = 0
#        last_len = 0 
#       
#    
#        while len(set(result.stats.keys())) < len(set([str(m) for m in result.motifs])):
#            if n >= 30:
#                logger.debug("waited long enough")
#                logger.debug("motifs: %s, stats: %s", len(result.motifs), len(result.stats.keys()))
#                for i,motif in enumerate(result.motifs):
#                    if "{}_{}".format(motif.id, motif.to_consensus()) not in result.stats:
#                        logger.debug("deleting %s", motif)
#                        del result.motifs[i]
#                break
#            sleep(2)
#            if len(result.stats.keys()) == last_len:
#                n += 1
#            else:
#                last_len = len(result.stats.keys())
#                n = 0
#    
    return result
628418
def parse_out(self, fo):
        
        motifs = []
        nucs = {"A":0,"C":1,"G":2,"T":3}
        pseudo = 0.0 # Should be 1/sqrt(# of seqs)
        aligns = {}
        for line in fo.readlines():
            if line.startswith("#"):
                pass
            elif len(line) > 1:
                vals = line.strip().split("\t")
                m_id, site = [x.strip().split(" ")[1].replace('"',"") for x in vals[8].split(";") if x]
                #if vals[6] == "+":
                if site.upper().find("N") == -1:
                    aligns.setdefault(m_id, []).append(site)
                #else:
                #    print site, rc(site)
                #    aligns.setdefault(id, []).append(rc(site))
                        
        for m_id, align in aligns.items():
            #print id, len(align)

            width = len(align[0])
            pfm =  [[0 for x in range(4)] for x in range(width)]
            for row in align:
                for i in range(len(row)):
                    pfm[i][nucs[row[i]]] += 1
            total = float(len(align))
            pwm = [[(x + pseudo/4)/total+(pseudo) for x in row] for row in pfm]
            m = Motif()
            m.align = align[:]
            m.pwm = pwm[:]
            m.pfm = pfm[:]
            m.id = m_id
            motifs.append(m)    
        return motifs
291412
def _ParseTokenType(self, file_object, file_offset):
    
    token_type_map = self._GetDataTypeMap('uint8')

    token_type, _ = self._ReadStructureFromFileObject(
        file_object, file_offset, token_type_map)

    return token_type
732951
def split_sequence_file_on_sample_ids_to_files(seqs,
                                               outdir):
    
    logger = logging.getLogger(__name__)
    logger.info('split_sequence_file_on_sample_ids_to_files'
                ' for file %s into dir %s' % (seqs, outdir))

    outputs = {}

    for bits in sequence_generator(seqs):
        sample = sample_id_from_read_id(bits[0])

        if sample not in outputs:
            outputs[sample] = open(join(outdir, sample + '.fasta'), 'w')
        outputs[sample].write(">%s\n%s\n" % (bits[0], bits[1]))
    for sample in outputs:
        outputs[sample].close()
    logger.info('split to %d files' % len(outputs))
1277236
def _fasta_slice(fasta, seqid, start, stop, strand):
    
    _strand = 1 if strand == '+' else -1
    return fasta.sequence({'chr': seqid, 'start': start, 'stop': stop, \
        'strand': _strand})
686048
def file_hash(load, fnd):
    '''
    
    '''
    if 'env' in load:
        # "env" is not supported; Use "saltenv".
        load.pop('env')

    if not all(x in load for x in ('path', 'saltenv')):
        return ''
    ret = {'hash_type': __opts__['hash_type']}
    relpath = fnd['rel']
    path = fnd['path']
    hashdest = os.path.join(__opts__['cachedir'],
                            'hgfs/hash',
                            load['saltenv'],
                            '{0}.hash.{1}'.format(relpath,
                                                  __opts__['hash_type']))
    if not os.path.isfile(hashdest):
        ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type'])
        with salt.utils.files.fopen(hashdest, 'w+') as fp_:
            fp_.write(ret['hsum'])
        return ret
    else:
        with salt.utils.files.fopen(hashdest, 'rb') as fp_:
            ret['hsum'] = salt.utils.stringutils.to_unicode(fp_.read())
        return ret
398452
def get_hash(path, cut=0):
    
    code, output = call(path, 'rev-parse', 'HEAD')
    if code > 0:
        return None
    # maybe the string is with a linebreak.
    sha1 = output.strip()
    if cut > 0:
        sha1 = sha1[:7]
    return sha1
1040872
def parse(self, fo):
        
        motifs = []
        nucs = {"A":0,"C":1,"G":2,"T":3}

        lines = fo.readlines()
        for i in range(0, len(lines), 5):
            align = []
            pwm = []
            pfm = []
            m_id = ""
            line = lines[i].strip()
            m_id = line[1:]
            number = m_id.split("_")[0][1:]
            if os.path.exists("%s.seq" % number):
                with open("%s.seq" % number) as f:
                    for l in f:
                        if "x" not in l and "n" not in l:
                            l = l.strip().upper()
                            align.append(l)
                            if not pfm:
                                pfm = [[0 for x in range(4)] for x in range(len(l))]
                            for p in range(len(l)):
                                pfm[p][nucs[l[p]]] += 1
    
            m = [l.strip().split(" ")[1].split("\t") for l in lines[i + 1: i + 5]]

            pwm = [[float(m[x][y]) for x in range(4)] for y in range(len(m[0]))]


            motifs.append(Motif(pwm))
            motifs[-1].id = "{}_{}".format(self.name, m_id)
            #motifs[-1].pwm = pwm
            if align:
                motifs[-1].pfm = pfm
                motifs[-1].align = align

        return motifs