OBO file retrieved from http://obo.cvs.sourceforge.net/viewvc/obo/obo/ontology/genomic-proteomic/so.obo
349592
def load_GODag():
    
    from jcvi.apps.base import download

    so_file_url = "http://obo.cvs.sourceforge.net/viewvc/obo/obo/ontology/genomic-proteomic/so.obo"
    so_file = download(so_file_url, debug=False)

    return GODag(so_file)
393963
def fetch():
    
    doi = '10.7910/DVN/VBSI4A'
    
    for component in ['dust', 'err']:
        requirements = {'filename': 'PG_{}_4096_ngp.fits'.format(component)}
        local_fname = os.path.join(
            data_dir(),
            'pg2010', 'PG_{}_4096_ngp.fits'.format(component))
        print('Downloading P&G (2010) {} data file to {}'.format(
            component, local_fname))
        fetch_utils.dataverse_download_doi(
            doi,
            local_fname,
            file_requirements=requirements)
1287264
def download_from_github(fname, path):
    
    base_url = 'https://github.com/ornlneutronimaging/ImagingReso/blob/master/ImagingReso/reference_data/'
    # Add GitHub junk to the file name for downloading.
    f = fname + '?raw=true'
    url = base_url + f
    block_size = 16384
    req = urlopen(url)

    # Get file size from header
    if sys.version_info[0] < 3:
        file_size = int(req.info().getheaders('Content-Length')[0])
    else:
        file_size = req.length
    # downloaded = 0

    # Check if file already downloaded
    if os.path.exists(fname):
        if os.path.getsize(fname) == file_size:
            print("Skipping downloading '{}'".format(fname))
        else:
            overwrite = input("File size changed, overwrite '{}'? ([y]/n) ".format(fname))
            if overwrite.lower().startswith('n'):
                print("Local file '{}' kept without overwriting.".format(fname))

    # Copy file to disk
    print("Downloading '{}'... ".format(fname))
    with open(fname, 'wb') as fh:
        while True:
            chunk = req.read(block_size)
            if not chunk:
                break
            fh.write(chunk)
            # downloaded += len(chunk)
        print('')
    print('Download completed.')
    print("Unzipping '{}'... ".format(fname))

    _database_zip = zipfile.ZipFile(fname)
    _database_zip.extractall(path=path)
    print("'{}' has been unzipped and database '{}' is ready to use.".format(fname, fname.replace('.zip', '')))

    os.remove(fname)
    print("'{}' has been deleted".format(fname))
1191582
def _extract_germline(in_file, data):
    
    out_file = "%s-germline.vcf" % utils.splitext_plus(in_file)[0]
    if not utils.file_uptodate(out_file, in_file) and not utils.file_uptodate(out_file + ".gz", in_file):
        with file_transaction(data, out_file) as tx_out_file:
            reader = cyvcf2.VCF(str(in_file))
            reader.add_filter_to_header({'ID': 'Somatic', 'Description': 'Variant called as Somatic'})
            #with contextlib.closing(cyvcf2.Writer(tx_out_file, reader)) as writer:
            with open(tx_out_file, "w") as out_handle:
                out_handle.write(reader.raw_header)
                for rec in reader:
                    rec = _update_germline_filters(rec)
                    out_handle.write(str(rec))
                    #writer.write_record(rec)
    return out_file
607505
def contents(self):
        
        code = "proc contents data=" + self.libref + '.' + self.table + self._dsopts() + ";run;"

        if self.sas.nosub:
            print(code)
            return

        ll = self._is_valid()
        if self.results.upper() == 'PANDAS':
            code  = "proc contents data=%s.%s %s ;" % (self.libref, self.table, self._dsopts())
            code += "ods output Attributes=work._attributes;"
            code += "ods output EngineHost=work._EngineHost;"
            code += "ods output Variables=work._Variables;"
            code += "ods output Sortedby=work._Sortedby;"
            code += "run;"
            return self._returnPD(code, ['_attributes', '_EngineHost', '_Variables', '_Sortedby'])

        else:
            if self.HTML:
                if not ll:
                    ll = self.sas._io.submit(code)
                if not self.sas.batch:
                    self.sas.DISPLAY(self.sas.HTML(ll['LST']))
                else:
                    return ll
            else:
                if not ll:
                    ll = self.sas._io.submit(code, "text")
                if not self.sas.batch:
                    print(ll['LST'])
                else:
                    return ll
1310298
function scrapDescription(window){
   var $ = window.$;
   var descriptions = [];

   // Open Graph protocol by Facebook <meta property="og:description" content="(*)"/>
   $('meta[property="og:description"]').each(function(){
      var content = $(this).attr('content');
      if(content) descriptions.push(content);
   });

   // Schema.org : <* itemprop="description">(*)</*>
   $('[itemprop="description"]').each(function(){
      var text = $(this).text();
      if(text) descriptions.push(text);
   });

   // Meta tag description: <meta property="description" content="(*)" />
   $('meta[name="description"]').each(function(){
      var description = utils.inline($(this).attr('content')).trim();
      if(description) descriptions.push(description);
   });

   // Random text in div and p tags. Oriented product informations
   if(descriptions.length === 0){
      $('div,p').each(function(){
         if( ($(this).attr('class') && $(this).attr('class').toLowerCase() === 'productdesc') || ($(this).attr('id') && $(this).attr('id').toLowerCase() === 'productdesc')){
            var description = utils.inline($(this).text()).trim();
            if(description) descriptions.push(description);
         }
      });
   }
   return descriptions;
}
85137
def load_just_in_time_genome_alignment(path, ref_spec, extensions=None,
                                       index_exts=None, fail_no_index=True,
                                       verbose=False):
    """"""
    if index_exts is None and fail_no_index:
      raise ValueError("Failure on no index specified for loading genome " +
                       "alignment, but no index extensions specified")

    extensions = __trim_extensions_dot(extensions)
    index_exts = __trim_extensions_dot(index_exts)

    partial_chrom_files = {}
    whole_chrom_files = {}
    for fn in os.listdir(path):
      pth = os.path.join(path, fn)
      if os.path.isfile(pth):
        base, ext = os.path.splitext(pth)
        ext = __trim_extension_dot(ext)
        if extensions is None or ext in extensions:
          idx_path = __find_index(pth, index_exts)
          if idx_path is None and fail_no_index:
            raise PyokitIOError("No index file for " + fn)
          chrom, start, end = __split_genomic_interval_filename(fn)
          assert((start is None and end is None) or
                 (start is not None and end is not None))
          if start is None:
            if chrom in whole_chrom_files:
              raise PyokitIOError("multiple files for chrom " + chrom)
            whole_chrom_files[chrom] = (pth, idx_path)
          else:
            k = (chrom, start, end)
            if k in partial_chrom_files:
              other_pth = partial_chrom_files[k][0]
              raise PyokitIOError("multiple files for " + str(k) + " --> " +
                                  pth + " and " + other_pth)
            partial_chrom_files[k] = (pth, idx_path)

    def factory(k):
      pth, idx = k
      return build_genome_alignment_from_file(pth, ref_spec, idx, verbose)

    return JustInTimeGenomeAlignment(whole_chrom_files, partial_chrom_files,
                                     factory)
777136
def _kbos_from_survey_sym_model_input_file(model_file):
    
    lines = storage.open_vos_or_local(model_file).read().split('\n')
    kbos = []
    for line in lines:
        if len(line) == 0 or line[0] == '#':  # skip initial column descriptors and the final blank line
            continue
        kbo = ephem.EllipticalBody()
        values = line.split()
        kbo.name = values[8]
        kbo.j = values[9]
        kbo.k = values[10]
        kbo._a = float(values[0])
        kbo._e = float(values[1])
        kbo._inc = float(values[2])
        kbo._Om = float(values[3])
        kbo._om = float(values[4])
        kbo._M = float(values[5])
        kbo._H = float(values[6])
        epoch = ephem.date(2453157.50000 - ephem.julian_date(0))
        kbo._epoch_M = epoch
        kbo._epoch = epoch
        kbos.append(kbo)
    return kbos
569219
def fetch(self, is_dl_forced=False):
        
        self.get_files(is_dl_forced)

        ncbi = NCBIGene(self.graph_type, self.are_bnodes_skized)
        # ncbi.fetch()
        gene_group = ncbi.files['gene_group']
        self.fetch_from_url(
            gene_group['url'], '/'.join((ncbi.rawdir, gene_group['file'])), False)

        # load and tag a list of OMIM IDs with types
        # side effect of populating omim replaced
        self.omim_type = self.find_omim_type()

        return
1057363
def download_ncbi_associations(gene2go="gene2go", prt=sys.stdout, loading_bar=True):
    """"""
    # Download: ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2go.gz
    gzip_file = "{GENE2GO}.gz".format(GENE2GO=gene2go)
    if not os.path.isfile(gene2go):
        file_remote = "ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/{GZ}".format(
            GZ=os.path.basename(gzip_file))
        dnld_file(file_remote, gene2go, prt, loading_bar)
    else:
        if prt is not None:
            prt.write("  EXISTS: {FILE}\n".format(FILE=gene2go))
    return gene2go
1159976
def liftover_to_genome(pass_pos, gtf):
    """"""

    fixed_pos = []
    for pos in pass_pos:
        if pos["chrom"] not in gtf:
            continue
        db_pos = gtf[pos["chrom"]][0]
        mut = _parse_mut(pos["sv"])
        print([db_pos, pos])
        if db_pos[3] == "+":
            pos['pre_pos'] = db_pos[1] + pos["pre_pos"] + 1
        else:
            pos['pre_pos'] = db_pos[2] - (pos["pre_pos"] - 1)
        pos['chrom'] = db_pos[0]
        pos['nt'] = list(mut[0])
        fixed_pos.append(pos)

    _print_header(fixed_pos)
    for pos in fixed_pos:
        print_vcf(pos)
378710
def data_from_diybio_org():
    """"""

    r = requests.get(diy_bio_labs_url)

    if r.status_code == 200:
        # Fix a problem in the html source while loading it
        data = BeautifulSoup(r.text.replace(u'\xa0', u''), "lxml")
    else:
        data = "There was an error while accessing data on diybio.org."

    return data
1320219
def fetch(self, is_dl_forced=True):
        

        username = config.get_config()['dbauth']['udp']['user']
        password = config.get_config()['dbauth']['udp']['password']

        credentials = (username, password)

        # Get patient map file:
        patient_id_map = self.open_and_parse_yaml(self.map_files['patient_ids'])
        udp_internal_ids = patient_id_map.keys()

        phenotype_fields = ['Patient', 'HPID', 'Present']

        # Get phenotype ids for each patient
        phenotype_params = {
            'method': 'search_subjects',
            'subject_type': 'Phenotype',
            'search_mode': 'DEEP',
            'fields': 'Patient',
            'conditions': 'equals',
            'values': ','.join(udp_internal_ids),
            'user_fields': ','.join(phenotype_fields)
        }

        prioritized_variants = [
            'Patient', 'Gene', 'Chromosome Position', 'Variant Allele', 'Transcript']

        prioritized_params = {
            'method': 'search_subjects',
            'subject_type': 'Variant Prioritization',
            'search_mode': 'DEEP',
            'fields': 'Patient',
            'conditions': 'equals',
            'values': ','.join(udp_internal_ids),
            'user_fields': ','.join(prioritized_variants),
            'format': 'json'}

        variant_fields = [
            'Patient', 'Family', 'Chr', 'Build', 'Chromosome Position',
            'Reference Allele', 'Variant Allele', 'Parent of origin',
            'Allele Type', 'Mutation Type', 'Gene', 'Transcript', 'Original Amino Acid',
            'Variant Amino Acid', 'Amino Acid Change', 'Segregates with',
            'Position', 'Exon', 'Inheritance model', 'Zygosity', 'dbSNP ID',
            '1K Frequency', 'Number of Alleles']

        variant_params = {
            'method': 'search_subjects',
            'subject_type': 'Exome Analysis Results',
            'search_mode': 'DEEP',
            'fields': 'Patient',
            'conditions': 'equals',
            'user_fields': ','.join(variant_fields),
            'format': 'json'}

        pheno_file = open(
            '/'.join((self.rawdir, self.files['patient_phenotypes']['file'])), 'w')
        variant_file = open(
            '/'.join((self.rawdir, self.files['patient_variants']['file'])), 'w')

        pheno_file.write('{0}\n'.format('\t'.join(phenotype_fields)))
        variant_file.write('{0}\n'.format('\t'.join(variant_fields)))

        variant_gene = self._fetch_data_from_udp(
            udp_internal_ids, prioritized_params, prioritized_variants, credentials)

        variant_gene_map = dict()
        for line in variant_gene:
            variant_gene_map.setdefault(line[0], []).append(
                # Try to make a unique value based on gene-pos-variantAlele-transcript
                # TODO make this a dict for readability purposes
                "{0}-{1}-{2}-{3}".format(line[1], line[2], line[3], line[4]))

        variant_info = self._fetch_data_from_udp(
            udp_internal_ids, variant_params, variant_fields, credentials)

        for line in variant_info:
            variant = "{0}-{1}-{2}-{3}".format(line[10], line[4], line[6], line[11])
            if variant in variant_gene_map[line[0]]:
                line[0] = patient_id_map[line[0]]
                line[4] = re.sub(r'\.0$', '', line[4])
                variant_file.write('{0}\n'.format('\t'.join(line)))

        phenotype_info = self._fetch_data_from_udp(
            udp_internal_ids, phenotype_params, phenotype_fields, credentials)

        for line in phenotype_info:
            line[0] = patient_id_map[line[0]]
            pheno_file.write('{0}\n'.format('\t'.join(line)))

        variant_file.close()
        pheno_file.close()

        return
656838
private String getSiteOU() {

        try {
            m_clonedCms.getRequestContext().setSiteRoot("");
            CmsResource resource = m_clonedCms.readResource(m_site.getSiteRoot());
            List<CmsRelation> relations = m_clonedCms.getRelationsForResource(resource, CmsRelationFilter.SOURCES);
            for (CmsRelation relation : relations) {
                if (relation.getSourcePath().startsWith("/system/orgunits/")) {
                    return (relation.getSourcePath().substring("/system/orgunits/".length()));
                }
            }
        } catch (CmsException e) {
            LOG.error("Error on reading OUs", e);
        }
        return "/";
    }
190793
def fetch_genome(genome_id):
    '''

    '''
    # TODO: Can strandedness by found in fetched genome attributes?
    # TODO: skip read/write step?
    # Using a dummy email for now - does this violate NCBI guidelines?
    email = 'loremipsum@gmail.com'
    Entrez.email = email

    print 'Downloading Genome...'
    handle = Entrez.efetch(db='nucleotide', id=str(genome_id), rettype='gb',
                           retmode='text')
    print 'Genome Downloaded...'
    tmpfile = os.path.join(mkdtemp(), 'tmp.gb')
    with open(tmpfile, 'w') as f:
        f.write(handle.read())
    genome = coral.seqio.read_dna(tmpfile)

    return genome
954657
def _get_gene2pubmed(self, limit):
        
        src_key = 'gene2pubmed'
        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        model = Model(graph)
        LOG.info("Processing Gene records")
        line_counter = 0
        myfile = '/'.join((self.rawdir, self.files[src_key]['file']))
        LOG.info("FILE: %s", myfile)
        assoc_counter = 0
        col = self.files[src_key]['columns']
        with gzip.open(myfile, 'rb') as tsv:
            row = tsv.readline().decode().strip().split('\t')
            row[0] = row[0][1:]  # strip comment
            if col != row:
                LOG.info(
                    '%s\nExpected Headers:\t%s\nRecived Headers:\t %s\n',
                    src_key, col, row)
            for line in tsv:
                line_counter += 1
                # skip comments
                row = line.decode().strip().split('\t')
                if row[0][0] == '#':
                    continue
                # (tax_num, gene_num, pubmed_num) = line.split('\t')

                # ## set id_filter=None in init if you don't want to have a filter
                # if self.id_filter is not None:
                #     if ((self.id_filter == 'taxids' and \
                #          (int(tax_num) not in self.tax_ids))
                #        or (self.id_filter == 'geneids' and \
                #            (int(gene_num) not in self.gene_ids))):
                #         continue
                # #### end filter

                gene_num = row[col.index('GeneID')].strip()
                if self.test_mode and int(gene_num) not in self.gene_ids:
                    continue

                tax_num = row[col.index('tax_id')].strip()
                if not self.test_mode and tax_num not in self.tax_ids:
                    continue

                pubmed_num = row[col.index('PubMed_ID')].strip()
                if gene_num == '-' or pubmed_num == '-':
                    continue

                gene_id = ':'.join(('NCBIGene', gene_num))
                pubmed_id = ':'.join(('PMID', pubmed_num))

                if self.class_or_indiv.get(gene_id) == 'C':
                    model.addClassToGraph(gene_id, None)
                else:
                    model.addIndividualToGraph(gene_id, None)
                # add the publication as a NamedIndividual
                # add type publication
                model.addIndividualToGraph(pubmed_id, None, None)
                reference = Reference(
                    graph, pubmed_id, self.globaltt['journal article'])
                reference.addRefToGraph()
                graph.addTriple(
                    pubmed_id, self.globaltt['is_about'], gene_id)
                assoc_counter += 1
                if not self.test_mode and limit is not None and line_counter > limit:
                    break

        LOG.info(
            "Processed %d pub-gene associations", assoc_counter)

        return
670565
def create_dexseq_annotation(gff, count_file):
    
    out_file = count_file + ".ann"
    if file_exists(out_file):
        return out_file
    with file_transaction(out_file) as tx_out:
        with open(tx_out, 'w') as out_handle:
            with open(gff) as in_handle:
                for line in in_handle:
                    cols = line.strip().split("\t")
                    if cols[2] == "exonic_part":
                        exon = [f for f in cols[8].split(";") if f.strip().startswith("exonic_part_number")]
                        gene = [f for f in cols[8].split(";") if f.strip().startswith("gene_id")]
                        exon = exon[0].replace("\"", "").split()[1]
                        gene = gene[0].replace("\"", "").split()[1]
                        length = int(cols[4]) - int(cols[3]) + 1
                        line = "%s:%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (gene, exon, gene,
                                                                    cols[0], cols[3],
                                                                    cols[4],
                                                                    length, cols[6])
                        out_handle.write(line)