Load an ontology formatted like Eidos' from github.
760742
def update_ontology(ont_url, rdf_path):
    """"""
    yaml_root = load_yaml_from_url(ont_url)
    G = rdf_graph_from_yaml(yaml_root)
    save_hierarchy(G, rdf_path)
717443
def local_imports(remote_base, local_base, ontologies, local_versions=tuple(), readonly=False, dobig=False, revert=False):
    
    done = []
    triples = set()
    imported_iri_vs_ontology_iri = {}
    p = owl.imports
    oi = b'owl:imports'
    oo = b'owl:Ontology'
    def inner(local_filepath, remote=False):
        if noneMembers(local_filepath, *bigleaves) or dobig:
            ext = os.path.splitext(local_filepath)[-1]
            if ext == '.ttl':
                infmt = 'turtle'
            else:
                print(ext, local_filepath)
                infmt = None
            if remote:
                resp = requests.get(local_filepath)  # TODO nonblocking pull these out, fetch, run inner again until done
                raw = resp.text.encode()
            else:
                try:
                    with open(local_filepath, 'rb') as f:
                        raw = f.read()
                except FileNotFoundError as e:
                    if local_filepath.startswith('file://'):
                        print('local_imports has already been run, skipping', local_filepath)
                        return
                        #raise ValueError('local_imports has already been run') from e
                    else:
                        print(e)  # TODO raise a warning if the file cannot be matched
                        # seems like good practice to have any imported ontology under
                        # version control so all imports are guaranteed to have good
                        # provenance and not split the prior informaiton between the
                        # scigraph config and the repository, the repository remains
                        # the source of truth, load.yaml files can then pick a subset
                        # of the properly tracked files to load as they see fit, but
                        # not add to them (at least in pyontutils land)
                        raw = b''
            if oo in raw:  # we only care if there are imports or an ontology iri
                scratch = rdflib.Graph()
                if infmt == 'turtle':
                    data, rest = raw.split(b'###', 1)
                elif infmt == None:  # assume xml
                    xml_tree = etree.parse(BytesIO(raw))
                    xml_root = xml_tree.getroot()
                    xml_ontology = xml_tree.xpath("/*[local-name()='RDF']/*[local-name()='Ontology']")
                    xml_root.clear()
                    xml_root.append(xml_ontology[0])
                    data = etree.tostring(xml_root)
                scratch.parse(data=data, format=infmt)
                for s in scratch.subjects(rdf.type, owl.Ontology):
                    triples.add((s, owl.sameAs, rdflib.URIRef(local_filepath)))
                    # somehow this breaks computing the chain
                    #for p in (rdfs.comment, skos.definition, definition, dc.title, rdfs.label):
                        #for o in scratch[s:p]:
                            #triples.add((s, p, o))
                for s, o in sorted(scratch.subject_objects(p)):
                    if revert:
                        raise NotImplemented('TODO')
                    nlfp = o.replace(remote_base, local_base)
                    triples.add((s, p, o))
                    if 'http://' in local_filepath or 'external' in local_filepath:  # FIXME what to do about https used inconsistently :/
                        if 'external' in local_filepath:
                            imported_iri = rdflib.URIRef(local_filepath.replace(local_base, remote_base))  # inefficient
                        else:
                            imported_iri = rdflib.URIRef(local_filepath)
                        if s != imported_iri:
                            imported_iri_vs_ontology_iri[imported_iri] = s  # kept for the record
                            triples.add((imported_iri, p, s))  # bridge imported != ontology iri
                    if local_base in nlfp and 'file://' not in o:  # FIXME file:// should not be slipping through here...
                        scratch.add((s, p, rdflib.URIRef('file://' + nlfp)))
                        scratch.remove((s, p, o))
                    if nlfp not in done:
                        done.append(nlfp)
                        if local_base in nlfp and 'external' not in nlfp:  # skip externals TODO
                            inner(nlfp)
                        elif readonly:  # read external imports
                            if 'external' in nlfp:
                                inner(nlfp)
                            else:
                                inner(nlfp, remote=True)
                if not readonly:
                    ttl = scratch.serialize(format='nifttl')
                    ndata, comment = ttl.split(b'###', 1)
                    out = ndata + b'###' + rest
                    with open(local_filepath, 'wb') as f:
                        f.write(out)

    for start in ontologies:
        print('START', start)
        done.append(start)
        inner(start)
    return sorted(triples)
1237423
def active_repositories(doc):
    """"""
    if doc.get('state') != 'deactivated':
        for repository_id, repo in doc.get('repositories', {}).items():
            if repo.get('state') != 'deactivated':
                repo['id'] = repository_id
                repo['organisation_id'] = doc['_id']

                yield repository_id, repo
366492
def docs(output=DOC_OUTPUT, proj_settings=PROJ_SETTINGS, github=False):
    

    local("export PYTHONPATH='' && "
          "export DJANGO_SETTINGS_MODULE=%s && "
          "sphinx-build -b html %s %s" % (proj_settings, DOC_INPUT, output),
          capture=False)

    if _parse_bool(github):
        local("touch %s/.nojekyll" % output, capture=False)
267057
def rdf_graph_from_yaml(yaml_root):
    """"""
    G = Graph()
    for top_entry in yaml_root:
        assert len(top_entry) == 1
        node = list(top_entry.keys())[0]
        build_relations(G, node, top_entry[node], None)
    return G
1246365
def loads(s, filename=None, loader=None, implicit_tuple=True, env={}, schema=None):
  """"""
  ast = reads(s, filename=filename, loader=loader, implicit_tuple=implicit_tuple)
  if not isinstance(env, framework.Environment):
    # For backwards compatibility we accept an Environment object. Otherwise assume it's a dict
    # whose bindings will add/overwrite the default bindings.
    env = framework.Environment(dict(_default_bindings, **env))
  obj = framework.eval(ast, env)
  return mod_schema.validate(obj, schema)
1175536
def load_yaml(filename):
    """"""
    with open(filename) as myfile:
        content = myfile.read()
        if "win" in sys.platform:
            content = content.replace("\\", "/")
        return yaml.safe_load(content)
758348
def load_schema(schema):
    
    with repo.Repo.db:
        repo.Repo.db.executescript(schema)
1080709
def load_repo_addons(_globals):
    '''
    '''
    repos_dir = os.path.expanduser('~/.fabsetup-addon-repos')
    if os.path.isdir(repos_dir):
        basedir, repos, _ = next(os.walk(repos_dir))
        for repo_dir in [os.path.join(basedir, repo)
                         for repo in repos
                         # omit dot dirs like '.rope'
                         # or 'fabsetup-theno-termdown.disabled'
                         if '.' not in repo]:
            sys.path.append(repo_dir)
            package_name, username = package_username(repo_dir.split('/')[-1])
            load_addon(username, package_name, _globals)
1224080
def Parse(self, stat, file_object, knowledge_base):
    """"""
    _, _ = stat, knowledge_base

    packages = []
    sw_data = utils.ReadFileBytesAsUnicode(file_object)
    try:
      for pkg in self._deb822.Packages.iter_paragraphs(sw_data.splitlines()):
        if self.installed_re.match(pkg["Status"]):
          packages.append(
              rdf_client.SoftwarePackage(
                  name=pkg["Package"],
                  description=pkg["Description"],
                  version=pkg["Version"],
                  architecture=pkg["Architecture"],
                  publisher=pkg["Maintainer"],
                  install_state="INSTALLED"))
    except SystemError:
      yield rdf_anomaly.Anomaly(
          type="PARSER_ANOMALY", symptom="Invalid dpkg status file")
    finally:
      if packages:
        yield rdf_client.SoftwarePackages(packages=packages)
1040492
def main():
    
    parser = argparse.ArgumentParser(description='Wrapper for obographs assocmodel library'
                                                 """
                                                 By default, ontologies and assocs are cached locally and synced from a remote sparql endpoint
                                                 """,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-r', '--resource', type=str, required=False,
                        help='Name of ontology')
    parser.add_argument('-f', '--file', type=str, required=False,
                        help='Name of input file for associations - currently GAF is assumed')
    parser.add_argument('-F', '--format', type=str, required=False,
                        help='Format of assoc file. One of GAF, GPAD or HPOA')
    parser.add_argument('-o', '--outfile', type=str, required=False,
                        help='Path to output file')
    parser.add_argument("--report-md", type=str, required=False, dest="report_md",
                        help="Path to report markdown file")
    parser.add_argument("--report-json", type=str, required=False, dest="report_json",
                        help="Path to report JSON file")
    parser.add_argument('-t', '--to', type=str, required=False,
                        help='Output to (tree, dot, ...)')
    parser.add_argument("--filter-out", nargs="+", required=False, default=[], metavar="EVIDENCE",
                        help="List of any evidence codes to filter out of the GAF. E.G. --filter-out IEA IMP")
    parser.add_argument("--filtered-file", required=False, default=None, metavar="FILTERED_FILE",
                        help="File to write the filtered out evidence GAF to")
    parser.add_argument('-T', '--taxon', nargs='*', required=False,
                        help='valid taxon (NCBITaxon ID) - validate against this')
    parser.add_argument('--subject_prefix', nargs='*', required=False,
                        help='E.g PomBase - validate against this')
    parser.add_argument('--object_prefix', nargs='*', required=False,
                        help='E.g GO - validate against this')
    parser.add_argument('-v', '--verbosity', default=0, action='count',
                        help='Increase output verbosity')


    subparsers = parser.add_subparsers(dest='subcommand', help='sub-command help')

    parser_n = subparsers.add_parser('validate', help='Validate associations')
    parser_n.set_defaults(function=validate_assocs)

    parser_n = subparsers.add_parser('filter', help='Filter associations')
    parser_n.set_defaults(function=filter_assocs)

    parser_n = subparsers.add_parser('convert', help='Convert associations')
    parser_n.set_defaults(function=convert_assocs)
    parser_n.add_argument('-t', '--to', type=str, required=True,
                          help='Format to convert to')

    parser_n = subparsers.add_parser('map2slim', help='Map to a subset/slim')
    parser_n.set_defaults(function=map2slim)
    parser_n.add_argument('-p', '--properties', nargs='*', type=str, default=['subClassOf', 'BFO:0000050'],
                          help='Properties')
    parser_n.add_argument('-s', '--subset', type=str, required=True,
                          help='subset (e.g. map2slim)')

    args = parser.parse_args()

    if args.verbosity >= 2:
        logging.basicConfig(level=logging.DEBUG)
    elif args.verbosity == 1:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARNING)

    logging.info("Welcome!")

    handle = args.resource

    # Ontology Factory
    ofactory = OntologyFactory()
    logging.info("Creating ont object from: {} {}".format(handle, ofactory))
    ont = ofactory.create(handle)
    logging.info("ont: {}".format(ont))

    func = args.function

    # Upper case all evidence codes
    args.filter_out = [code.upper() for code in args.filter_out]

    # set configuration
    filtered_evidence_file = open(args.filtered_file, "w") if args.filtered_file else None
    config = assocparser.AssocParserConfig(
        valid_taxa=args.taxon,
        ontology=ont,
        class_idspaces=args.object_prefix,
        entity_idspaces=args.subject_prefix,
        filter_out_evidence=args.filter_out,
        filtered_evidence_file=filtered_evidence_file
    )
    p = None
    fmt = None
    if args.format is None:
        fmt = 'gaf'
    else:
        fmt = args.format.lower()

    # TODO: use a factory
    if fmt == 'gaf':
        p = GafParser(dataset=args.file)
    elif fmt == 'gpad':
        p = GpadParser()
    elif fmt == 'hpoa':
        p = HpoaParser()
    elif fmt == "gpi":
        p = entityparser.GpiParser()
        func = validate_entity

    p.config = config

    outfh = None
    if args.outfile is not None:
        two_mb = 2097152
        outfh = open(args.outfile, "w", buffering=two_mb)
    func(ont, args.file, outfh, p, args)
    if filtered_evidence_file:
        filtered_evidence_file.close()

    if outfh is not None:
        outfh.close()

    if args.report_md is not None:
        report_md = open(args.report_md, "w")
        report_md.write(p.report.to_markdown())
        report_md.close()
    if args.report_json is not None:
        report_json = open(args.report_json, "w")
        report_json.write(json.dumps(p.report.to_report_json(), indent=4))
        report_json.close()
    if not (args.report_md or args.report_json):
        print(p.report.to_markdown())
623104
def repositories(doc):
    """"""
    for repository_id, repo in doc.get('repositories', {}).items():
        repo['id'] = repository_id
        repo['organisation_id'] = doc['_id']

        yield repository_id, repo
481125
def loadtoc(self):
        """"""
        self.toc = self.TOCTMPLT()
        self.lib.seek(self.pkgstart+self.tocpos)
        tocstr = self.lib.read(self.toclen)
        self.toc.frombinary(tocstr)
743333
def projects_from_metadata(metadata):
    """"""
    projects = []
    for data in metadata:
        meta = distlib.metadata.Metadata(fileobj=io.StringIO(data))
        projects.extend(pypi.just_name(project) for project in meta.run_requires)
    return frozenset(map(packaging.utils.canonicalize_name, projects))
445731
def _build_rdf(self, data=None):

		'''
		
		'''

		# recreate rdf data
		self.rdf = SimpleNamespace()
		self.rdf.data = data
		self.rdf.prefixes = SimpleNamespace()
		self.rdf.uris = SimpleNamespace()
		# populate prefixes
		for prefix,uri in self.repo.context.items():
			setattr(self.rdf.prefixes, prefix, rdflib.Namespace(uri))
		# graph
		self._parse_graph()
930397
def load_requirements(self):
        """"""

        for module_name, pip_name in iteritems(self.metadata.requirements):
            extant = self.dataset.config.requirements[module_name].url

            force = (extant and extant != pip_name)

            self._library.install_packages(module_name, pip_name, force=force)

            self.dataset.config.requirements[module_name].url = pip_name

        python_dir = self._library.filesystem.python()
        sys.path.append(python_dir)
321816
def _load_documentation(self, term, contents, file_name):
        """"""

        try:
            title = term['title'].value
        except KeyError:
            self.warn("Documentation has no title, skipping: '{}' ".format(term.value))
            return

        if term.term_is('Root.Readme'): # This term type has inline content, not a url
            package_sub_dir = 'docs'
        else:
            try:
                eu = term.expanded_url
                parsed_url = term.parsed_url
            except AttributeError:
                parsed_url = eu = parse_app_url(term.value)

            # Can't used expanded_url here because expansion makes file system URLS absolute.
            if eu.proto == 'file' and not parsed_url.path_is_absolute:
                package_sub_dir = parsed_url.fspath.parent
            else:
                package_sub_dir = 'docs'


        path = join(self.package_path.path,  package_sub_dir, file_name)

        self.prt("Loading documentation for '{}', '{}' to '{}'  ".format(title, file_name, path))

        makedirs(dirname(path), exist_ok=True)

        if exists(path):
            remove(path)

        with open(path, 'wb') as f:
            f.write(contents)