Download, extract and import unihan to database.
512940
def bootstrap_unihan(metadata, options={}):
    """"""
    options = merge_dict(UNIHAN_ETL_DEFAULT_OPTIONS.copy(), options)

    p = unihan.Packager(options)
    p.download()
    data = p.export()
    table = create_unihan_table(UNIHAN_FIELDS, metadata)
    metadata.create_all()
    metadata.bind.execute(table.insert(), data)
610222
def download_unicodedata(version, output=HOME, no_zip=False):
    """"""
    files = [
        'UnicodeData.txt',
        'Scripts.txt',
        'Blocks.txt',
        'PropList.txt',
        'DerivedCoreProperties.txt',
        'DerivedNormalizationProps.txt',
        'CompositionExclusions.txt',
        'PropertyValueAliases.txt',
        'PropertyAliases.txt',
        'EastAsianWidth.txt',
        'LineBreak.txt',
        'HangulSyllableType.txt',
        'DerivedAge.txt',
        'auxiliary/WordBreakProperty.txt',
        'auxiliary/SentenceBreakProperty.txt',
        'auxiliary/GraphemeBreakProperty.txt',
        'extracted/DerivedDecompositionType.txt',
        'extracted/DerivedNumericType.txt',
        'extracted/DerivedNumericValues.txt',
        'extracted/DerivedJoiningType.txt',
        'extracted/DerivedJoiningGroup.txt',
        'extracted/DerivedCombiningClass.txt'
    ]

    files.append('ScriptExtensions.txt')
    if PY35:
        files.append('IndicPositionalCategory.txt')
    else:
        files.append('IndicMatraCategory.txt')
    files.append('IndicSyllabicCategory.txt')

    if PY34:
        files.append('BidiBrackets.txt')

    if PY37:
        files.append('VerticalOrientation.txt')

    http_url = 'http://www.unicode.org/Public/%s/ucd/' % version
    ftp_url = 'ftp://ftp.unicode.org/Public/%s/ucd/' % version

    destination = os.path.join(output, 'unicodedata', version)
    if not os.path.exists(destination):
        os.makedirs(destination)

    zip_data = not no_zip

    for f in files:
        file_location = os.path.join(destination, os.path.basename(f))
        retrieved = False
        if not os.path.exists(file_location):
            for url in (ftp_url, http_url):
                furl = url + f
                try:
                    print('Downloading: %s --> %s' % (furl, file_location))
                    response = urlopen(furl, timeout=30)
                    data = response.read()
                except Exception:
                    print('Failed: %s' % url)
                    continue
                with open(file_location, 'w') as uf:
                    uf.write(data.decode('utf-8'))
                retrieved = True
                break
            if not retrieved:
                print('Failed to acquire all needed Unicode files!')
                break
        else:
            retrieved = True
            print('Skipping: found %s' % file_location)

        if not retrieved:
            zip_data = False
            break

    if zip_data and not os.path.exists(os.path.join(output, 'unicodedata', '%s.zip' % version)):
        zip_unicode(output, version)
526276
def _load(db_data, db):
    
    if db.name != db_data["name"]:
        raise ValueError("dbname doesn't matches! Maybe wrong database data.")

    db.__init__(client=db._client, name=db.name)
    for col_name, col_data in iteritems(db_data["_collections"]):
        collection = db.get_collection(col_name)
        collection._documents = col_data["_documents"]
        collection._uniques = col_data["_uniques"]
        db._collections[col_name] = collection

    return db
141968
def download_seq_file(self, outdir, force_rerun=False):
        """"""

        uniprot_fasta_file = download_uniprot_file(uniprot_id=self.id,
                                                   filetype='fasta',
                                                   outdir=outdir,
                                                   force_rerun=force_rerun)

        self.sequence_path = uniprot_fasta_file
1195356
def import_model(self, source):
        """"""
        source = self._resolve_source(source)
        self._context = FilePathContext(source)
        with self._context.open() as f:
            self._reader = self._open_reader(f)

        return self._reader.create_model()
391254
def convert_old(data):
    ""
    ret = default_config()
    ret['sync'].update(data.get('peeringdb', {}))
    ret['orm']['database'].update(data.get('database', {}))
    return ret
442679
def import_handle(self, original, loc, tokens):
        """"""
        if len(tokens) == 1:
            imp_from, imports = None, tokens[0]
        elif len(tokens) == 2:
            imp_from, imports = tokens
            if imp_from == "__future__":
                self.strict_err_or_warn("unnecessary from __future__ import (Coconut does these automatically)", original, loc)
                return ""
        else:
            raise CoconutInternalException("invalid import tokens", tokens)
        if self.strict:
            self.unused_imports.update(imported_names(imports))
        return universal_import(imports, imp_from=imp_from, target=self.target)
497258
def download(config, account, day, region, output):
    """"""

    with open(config) as fh:
        config = yaml.safe_load(fh.read())

    jsonschema.validate(config, CONFIG_SCHEMA)

    found = None
    for info in config['accounts']:
        if info['name'] == account:
            found = info
            break

    if not found:
        log.info("Account %s not found", account)
        return

    s3 = boto3.client('s3')
    day = parse_date(day)

    key_data = dict(found)
    key_data['region'] = region
    key_data['date_fmt'] = "%s/%s/%s" % (
        day.year, day.month, day.day)
    key = config['key_template'] % key_data

    s3.download_file(found['bucket'], key, output + '.bz2')
    subprocess.check_call(["lbzip2", "-d", output + '.bz2'])
1023713
def from_dict(cls, database, key, data, clear=False):
        
        hsh = cls(database, key)
        if clear:
            hsh.clear()
        hsh.update(data)
        return hsh
1316744
def fetch(force=False):
    
    try:
        # python 2
        from urllib2 import urlopen, HTTPError, URLError
    except ImportError:
        # python 3
        from urllib.request import urlopen, HTTPError, URLError

    if os.path.isdir(FIJI_ROOT) and not force:
        return
    elif not os.path.isdir(FIJI_ROOT):
        print('Fiji missing in %s' % FIJI_ROOT)

    if force:
        print('Deleting %s' % FIJI_ROOT)
        shutil.rmtree(FIJI_ROOT, ignore_errors=True)

    print('Downloading fiji from %s' % URL)
    try:
        req = urlopen(URL)
        try:
            size = int(req.info()['content-length'])
        except AttributeError:
            size = -1

        chunk = 512*1024
        fp = BytesIO()
        i = 0
        while 1:
            data = req.read(chunk)
            if not data:
                break
            fp.write(data)
            if size > 0:
                percent = fp.tell() // (size/100)
                msg = 'Downloaded %d percent      \r' % percent
            else:
                msg = 'Downloaded %d bytes\r' % fp.tell()
            sys.stdout.write(msg)
    except (HTTPError, URLError) as e:
        print('Error getting fiji: {}'.format(e))
        sys.exit(1)

    try:
        print('\nExtracting zip')
        z = ZipFile(fp)
        z.extractall(BIN_FOLDER)
        # move to Fiji-VERSION.app to easily check if it exists (upon fijibin upgrade)
        os.rename(EXTRACT_FOLDER, FIJI_ROOT)
    except (BadZipFile, IOError) as e:
        print('Error extracting zip: {}'.format(e))
        sys.exit(1)

    for path in BIN_NAMES.values():
        st = os.stat(path)
        os.chmod(path, st.st_mode | stat.S_IEXEC)
799037
def update(connection, force_download):
    """"""
    manager.database.update(
        connection=connection,
        force_download=force_download
    )
505315
def import_sqlite(db_file, older_than=None, **kwargs):
    """"""
    conn = _open_sqlite(db_file)
    cur = conn.cursor()
    # get rows that were not exported yet
    select = "SELECT * FROM testcases WHERE exported != 'yes'"
    if older_than:
        cur.execute(" ".join((select, "AND sqltime < ?")), (older_than,))
    else:
        cur.execute(select)
    columns = [description[0] for description in cur.description]
    rows = cur.fetchall()

    # map data to columns
    results = []
    for row in rows:
        record = OrderedDict(list(zip(columns, row)))
        results.append(record)

    testrun = _get_testrun_from_sqlite(conn)

    conn.close()

    return xunit_exporter.ImportedData(results=results, testrun=testrun)
374266
def do_import(token, account_uuid, bank_account, since=None):
    
    response = requests.get(
        url="https://api.teller.io/accounts/{}/transactions".format(account_uuid),
        headers={"Authorization": "Bearer {}".format(token)},
    )
    response.raise_for_status()
    data = response.json()

    statement_import = StatementImport.objects.create(
        source="teller.io", extra={"account_uuid": account_uuid}, bank_account=bank_account
    )

    for line_data in data:
        uuid = UUID(hex=line_data["id"])
        if StatementLine.objects.filter(uuid=uuid):
            continue

        description = ", ".join(filter(bool, [line_data["counterparty"], line_data["description"]]))
        date = datetime.date(*map(int, line_data["date"].split("-")))

        if not since or date >= since:
            StatementLine.objects.create(
                uuid=uuid,
                date=line_data["date"],
                statement_import=statement_import,
                amount=line_data["amount"],
                type=line_data["type"],
                description=description,
                source_data=line_data,
            )
1366281
def from_eocube(eocube, ji):
        """"""
        eocubewin = EOCubeChunk(ji, eocube.df_layers, eocube.chunksize, eocube.wdir)
        return eocubewin
1296545
def update(taxids, conn, force_download, silent):
    """"""
    if not silent:
        click.secho("WARNING: Update is very time consuming and can take several "
                    "hours depending which organisms you are importing!", fg="yellow")

        if not taxids:
            click.echo("Please note that you can restrict import to organisms by "
                       "NCBI taxonomy IDs")
            click.echo("Example (human, mouse, rat):\n")
            click.secho("\tpyuniprot update --taxids 9606,10090,10116\n\n", fg="green")

    if taxids:
        taxids = [int(taxid.strip()) for taxid in taxids.strip().split(',') if re.search('^ *\d+ *$', taxid)]

    database.update(taxids=taxids, connection=conn, force_download=force_download, silent=silent)
1298151
def export(self, db):
        
        dbnode = ET.Element('database')
    
        if self.include_comment:
            now = datetime.now()
            filepath = db.filepath
            if filepath:
                comment = ET.Comment('Generated by keepassdb from {0} on {1}'.format(filepath, now.strftime("%c")))
            else:
                comment = ET.Comment('Generated by keepassdb on {0}'.format(now.strftime("%c")))
            dbnode.append(comment)
        
        def _date(dt):
            if dt == const.NEVER:
                return 'Never'
            else:
                # 2012-12-20T20:56:56
                return dt.strftime('%Y-%m-%dT%H:%M:%S')
            
        def group_to_xml(group, node):
            gnode = ET.SubElement(node, 'group')
            title = ET.SubElement(gnode, 'title')
            title.text = group.title
            icon = ET.SubElement(gnode, 'icon')
            icon.text = str(group.icon)
            
            for subgroup in group.children:
                group_to_xml(subgroup, gnode)
            
            for entry in group.entries:
                if entry.title == 'Meta-Info' and entry.username == 'SYSTEM':
                    continue
                enode = ET.SubElement(gnode, 'entry')
                ET.SubElement(enode, 'title').text = entry.title
                ET.SubElement(enode, 'username').text = entry.username
                ET.SubElement(enode, 'password').text = entry.password
                ET.SubElement(enode, 'url').text = entry.url
                ET.SubElement(enode, 'comment').text = entry.notes
                ET.SubElement(enode, 'icon').text = str(entry.icon)
                ET.SubElement(enode, 'creation').text = _date(entry.created)
                ET.SubElement(enode, 'lastaccess').text = _date(entry.accessed)
                ET.SubElement(enode, 'lastmod').text = _date(entry.modified)
                ET.SubElement(enode, 'expire').text = _date(entry.expires)
            return gnode
        
        for group in db.root.children:
            dbnode.append(group_to_xml(group, dbnode))
            
        xmlstr = ET.tostring(dbnode)
        if self.prettyprint:
            reparsed = minidom.parseString(xmlstr)
            xmlstr = reparsed.toprettyxml(indent=" ")
        
        return xmlstr
840937
def import_from_dict(cls, session, dict_rep, parent=None,
                         recursive=True, sync=[]):
        """"""
        parent_refs = cls._parent_foreign_key_mappings()
        export_fields = set(cls.export_fields) | set(parent_refs.keys())
        new_children = {c: dict_rep.get(c) for c in cls.export_children
                        if c in dict_rep}
        unique_constrains = cls._unique_constrains()

        filters = []  # Using these filters to check if obj already exists

        # Remove fields that should not get imported
        for k in list(dict_rep):
            if k not in export_fields:
                del dict_rep[k]

        if not parent:
            if cls.export_parent:
                for p in parent_refs.keys():
                    if p not in dict_rep:
                        raise RuntimeError(
                            '{0}: Missing field {1}'.format(cls.__name__, p))
        else:
            # Set foreign keys to parent obj
            for k, v in parent_refs.items():
                dict_rep[k] = getattr(parent, v)

        # Add filter for parent obj
        filters.extend([getattr(cls, k) == dict_rep.get(k)
                        for k in parent_refs.keys()])

        # Add filter for unique constraints
        ucs = [and_(*[getattr(cls, k) == dict_rep.get(k)
               for k in cs if dict_rep.get(k) is not None])
               for cs in unique_constrains]
        filters.append(or_(*ucs))

        # Check if object already exists in DB, break if more than one is found
        try:
            obj_query = session.query(cls).filter(and_(*filters))
            obj = obj_query.one_or_none()
        except MultipleResultsFound as e:
            logging.error('Error importing %s \n %s \n %s', cls.__name__,
                          str(obj_query),
                          yaml.safe_dump(dict_rep))
            raise e

        if not obj:
            is_new_obj = True
            # Create new DB object
            obj = cls(**dict_rep)
            logging.info('Importing new %s %s', obj.__tablename__, str(obj))
            if cls.export_parent and parent:
                setattr(obj, cls.export_parent, parent)
            session.add(obj)
        else:
            is_new_obj = False
            logging.info('Updating %s %s', obj.__tablename__, str(obj))
            # Update columns
            for k, v in dict_rep.items():
                setattr(obj, k, v)

        # Recursively create children
        if recursive:
            for c in cls.export_children:
                child_class = cls.__mapper__.relationships[c].argument.class_
                added = []
                for c_obj in new_children.get(c, []):
                    added.append(child_class.import_from_dict(session=session,
                                                              dict_rep=c_obj,
                                                              parent=obj,
                                                              sync=sync))
                # If children should get synced, delete the ones that did not
                # get updated.
                if c in sync and not is_new_obj:
                    back_refs = child_class._parent_foreign_key_mappings()
                    delete_filters = [getattr(child_class, k) ==
                                      getattr(obj, back_refs.get(k))
                                      for k in back_refs.keys()]
                    to_delete = set(session.query(child_class).filter(
                        and_(*delete_filters))).difference(set(added))
                    for o in to_delete:
                        logging.info('Deleting %s %s', c, str(obj))
                        session.delete(o)

        return obj