Assemble file paths from tax ids
        Args:
            :param tax_ids (list) list of taxa
        Returns:
            :return file dict
345566
def _get_file_paths(self, tax_ids, file_type):
        
        file_paths = dict()
        if file_type not in self.files:
            raise KeyError("file type {} not configured".format(file_type))
        for taxon in tax_ids:
            file_paths[taxon] = {
                'file': "{}.{}".format(taxon, self.files[file_type]['pattern']),
                'url': "{}{}.{}".format(
                    self.files[file_type]['path'], taxon,
                    self.files[file_type]['pattern']),
                'headers': {'User-Agent': USER_AGENT}
            }
        return file_paths
846281
def chain_output_files(self):
        
        ret_list = []
        for key, val in self.file_dict.items():
            # For pure input files we only want output files that were not
            # marked as internal or temp
            if val & FileFlags.out_ch_mask == FileFlags.output_mask:
                ret_list.append(key)
        return ret_list
1020297
def from_taxdb(cls, con, root=None):
        
        cursor = con.cursor()
        if root is None:
            cursor.execute(
                "SELECT tax_id, rank FROM nodes WHERE tax_id = parent_id")
        else:
            cursor.execute(
                "SELECT tax_id, rank FROM nodes WHERE tax_id = ?", [root])

        tax_id, rank = cursor.fetchone()
        root = cls(rank=rank, tax_id=tax_id)

        def add_lineage(parent):
            cursor.execute("""SELECT tax_id, rank, tax_name
                    FROM nodes INNER JOIN names USING (tax_id)
                    WHERE parent_id = :1 and tax_id <> :1
                        AND names.is_primary = 1
                    """, [parent.tax_id])
            for tax_id, rank, name in cursor:
                node = cls(rank=rank, tax_id=tax_id, name=name)
                parent.add_child(node)
            for child in parent.children:
                add_lineage(child)

        add_lineage(root)
        return root
1058627
def fill_taxid2asscs(taxid2asscs_usr, taxid2asscs_ret):
        """"""
        for taxid, ab_ret in taxid2asscs_ret.items():
            taxid2asscs_usr[taxid]['ID2GOs'] = ab_ret['ID2GOs']
            taxid2asscs_usr[taxid]['GO2IDs'] = ab_ret['GO2IDs']
351684
def append_to_tables(self, table, table_ids):
        
        infile_refs = np.zeros((2), int)
        outfile_refs = np.zeros((2), int)
        rmfile_refs = np.zeros((2), int)
        intfile_refs = np.zeros((2), int)
        f_ptr = len(table_ids['file_id'])
        infile_refs[0] = f_ptr
        if self.infile_ids is not None:
            for fid in self.infile_ids:
                table_ids.add_row(dict(file_id=fid))
                f_ptr += 1
        infile_refs[1] = f_ptr
        outfile_refs[0] = f_ptr
        if self.outfile_ids is not None:
            for fid in self.outfile_ids:
                table_ids.add_row(dict(file_id=fid))
                f_ptr += 1
        outfile_refs[1] = f_ptr
        rmfile_refs[0] = f_ptr
        if self.rmfile_ids is not None:
            for fid in self.rmfile_ids:
                table_ids.add_row(dict(file_id=fid))
                f_ptr += 1
        rmfile_refs[1] = f_ptr
        intfile_refs[0] = f_ptr
        if self.intfile_ids is not None:
            for fid in self.intfile_ids:
                table_ids.add_row(dict(file_id=fid))
                f_ptr += 1
        intfile_refs[1] = f_ptr

        table.add_row(dict(dbkey=self.dbkey,
                           jobname=self.jobname,
                           jobkey=self.jobkey,
                           appname=self.appname,
                           logfile=self.logfile,
                           job_config=str(self.job_config),
                           timestamp=self.timestamp,
                           infile_refs=infile_refs,
                           outfile_refs=outfile_refs,
                           rmfile_refs=rmfile_refs,
                           intfile_refs=intfile_refs,
                           status=self.status))
864495
def batch_taxids(list_of_names):
    
    for name in list_of_names:
        handle = Entrez.esearch(db='Taxonomy', term=name, retmode="xml")
        records = Entrez.read(handle)
        yield records["IdList"][0]
4107
def nary_subtree(self, tax_id, n=2):
        
        if tax_id is None:
            return None
        parent_id, rank = self._node(tax_id)
        if rank == 'species':
            return [tax_id]
        else:
            children = self.children_of(tax_id, 2)
            species_taxids = []
            for t in children:
                species_taxids.extend(self.nary_subtree(t, n))
            return species_taxids
1103962
def iter_doc_filepaths(self, **kwargs):
        
        for shard in self._shards:
            for doc_id, blob in shard.iter_doc_filepaths(**kwargs):
                yield doc_id, blob
938895
def add_ds_ids_from_files(self):
        """"""
        for file_handlers in self.file_handlers.values():
            try:
                fh = file_handlers[0]
                avail_ids = fh.available_datasets()
            except NotImplementedError:
                continue

            # dynamically discover other available datasets
            for ds_id, ds_info in avail_ids:
                # don't overwrite an existing dataset
                # especially from the yaml config
                coordinates = ds_info.get('coordinates')
                if isinstance(coordinates, list):
                    # xarray doesn't like concatenating attributes that are
                    # lists: https://github.com/pydata/xarray/issues/2060
                    ds_info['coordinates'] = tuple(ds_info['coordinates'])
                self.ids.setdefault(ds_id, ds_info)
329600
def get_children(engine, parent_ids, rank='species', schema=None):
    

    if not parent_ids:
        return []

    nodes = schema + '.nodes' if schema else 'nodes'
    names = schema + '.names' if schema else 'names'

    cmd = ('select tax_id, tax_name, rank '
           'from {} join {} using (tax_id) '
           'where parent_id = :tax_id and is_primary').format(nodes, names)

    species = []
    for parent_id in parent_ids:
        result = engine.execute(sqlalchemy.sql.text(cmd), tax_id=parent_id)
        keys = list(result.keys())
        rows = [dict(list(zip(keys, row))) for row in result.fetchall()]
        for r in rows:
            if r['rank'] == rank and 'sp.' not in r['tax_name']:
                species.append(r)
        others = [r for r in rows if r['rank'] not in (rank, 'no_rank')]
        if others:
            _, s = get_children(engine, [r['tax_id'] for r in others])
            species.extend(s)

    return keys, species
1145392
def file_dict(*packages, **kwargs):
    '''
    
    '''
    errors = []
    files = {}

    for package in packages:
        cmd = ['pkg_info', '-qL', package]
        ret = __salt__['cmd.run_all'](cmd, output_loglevel='trace')
        files[package] = []
        for line in ret['stderr'].splitlines():
            errors.append(line)

        for line in ret['stdout'].splitlines():
            if line.startswith('/'):
                files[package].append(line)
            else:
                continue  # unexpected string

    ret = {'errors': errors, 'files': files}
    for field in list(ret):
        if not ret[field] or ret[field] == '':
            del ret[field]
    return ret
654107
def get_names(list_of_taxids):
    
    from jcvi.apps.fetch import batch_taxonomy
    list_of_taxids = [str(x) for x in list_of_taxids]
    return list(batch_taxonomy(list_of_taxids))
331047
def taxtable_to_tree(handle):
    """"""
    c = csv.reader(handle, quoting=csv.QUOTE_NONNUMERIC)
    header = next(c)
    rootdict = dict(list(zip(header, next(c))))
    t = Tree(rootdict['tax_id'], rank=rootdict[
             'rank'], tax_name=rootdict['tax_name'])
    for l in c:
        d = dict(list(zip(header, l)))
        target = t.descendents[d['parent_id']]
        target(Tree(d['tax_id'], rank=d['rank'], tax_name=d['tax_name']))
    return t
1204663
def collectfiles(path, filt=None):
    
    if not os.path.isdir(path):
        path = os.path.dirname(path)

    output = []
    for name in sorted(os.listdir(path)):
        filepath = os.path.join(path, name)
        if os.path.isfile(filepath):
            if not filt or filt(name):
                output.append((name, filepath))
    return output
1051154
def _init_taxid2asscs(self):
        """"""
        taxid2asscs = cx.defaultdict(list)
        for ntanno in self.associations:
            taxid2asscs[ntanno.tax_id].append(ntanno)
        assert len(taxid2asscs) != 0, "**FATAL: NO TAXIDS: {F}".format(F=self.filename)
        # """Print the number of taxids stored."""
        prt = sys.stdout
        num_taxids = len(taxid2asscs)
        prt.write('{N} taxids stored'.format(N=num_taxids))
        if num_taxids < 5:
            prt.write(': {Ts}'.format(Ts=' '.join(sorted(str(t) for t in taxid2asscs))))
        prt.write('\n')
        return dict(taxid2asscs)
422959
def taxTree(taxdict):
    """"""
    # the taxonomic dictionary holds the lineage of each ident in
    #  the same order as the taxonomy
    # use hierarchy to construct a taxonomic tree
    for rank in taxdict.taxonomy:
        current_level = float(taxdict.taxonomy.index(rank))
        # get clades at this rank in hierarchy
        clades = taxdict.hierarchy[rank]
        # merge those that are in the same clade into a cladestring
        for clade in clades:
            # unpack the identities in this clade and its clade name
            cladeidents, cladename = clade
            # Remove '' TaxRefs -- in cladestring already
            cladeidents = [e for e in cladeidents if e.ident]
            # only create cladestring if more than one ident in clade
            if len(cladeidents) < 2:
                continue
            # label node by 'clade'_'rank'
            cladename = '{0}_{1}'.format(cladename, rank)
            cladestring = stringClade(cladeidents, cladename, current_level)
            # replace first TaxRef in cladeidents with cladestring
            cladeidents[0].change(ident=cladestring, rank=rank)
            # replace all other TaxRefs with ''
            for e in cladeidents[1:]:
                e.change(ident='', rank=rank)
    # join any remaining strands into tree
    if len(taxdict.hierarchy[taxdict.taxonomy[-1]]) > 1:
        # unlist first
        clade = [e[0] for e in taxdict.hierarchy[taxdict.taxonomy[-1]]]
        cladeidents = sum(clade, [])
        cladeidents = [e for e in cladeidents if e.ident]
        cladestring = stringClade(cladeidents, 'life', current_level+1)
    return cladestring + ';'
579653
def filter_taxa(records, taxids, unclassified=False, discard=False):
    '''
     
    '''
    taxids = set(taxids)
    kept_records = []
    ncbi = ete3.NCBITaxa()
    unclassified_taxids = {0,1}
    for r in records:
        taxid, rank, sciname, lineage = r.description.strip().partition(' ')[2].split('|')
        taxid = int(taxid)
        lineage = set(ncbi.get_lineage(taxid) or [0])  # lineage defined twice?
        intersection = lineage & taxids
        if taxid in unclassified_taxids and unclassified:
            kept_records.append(r)
        elif intersection and not discard:
            kept_records.append(r)
        elif not intersection and discard and taxid not in unclassified_taxids:
            kept_records.append(r)

    return kept_records