Convert things on the way from the database to Python.
767861
def process_result_value(self, value: Optional[str],
                             dialect: Dialect) -> List[int]:
        """"""
        retval = self._dbstr_to_intlist(value)
        return retval
618001
def to_pandas(self, wrap=False, **kwargs):
        

        try:
            import pandas as pd
        except ImportError:
            raise DependencyNotInstalledError(
                    'to_pandas requires `pandas` library')

        def wrapper(result):
            res = result.values
            if wrap:
                from .. import DataFrame
                return DataFrame(res, schema=self.schema)
            return res

        return self.execute(wrapper=wrapper, **kwargs)
1051532
def to_python(self):
        
        if isinstance(self.data, str):
            return self.data.strip().lower() == 'true'
        if isinstance(self.data, int):
            return self.data > 0
        return bool(self.data)
517796
def to_python(self, value):
        """"""
        if isinstance(value, DirDescriptor):
            return value
        elif isinstance(value, str):
            return DirDescriptor(value)
        elif isinstance(value, dict):
            try:
                path = value['dir']
            except KeyError:
                raise ValidationError("dictionary must contain a 'dir' element")

            if not isinstance(path, str):
                raise ValidationError("field's dir element must be a string")

            size = value.get('size', None)
            if size is not None and not isinstance(size, int):
                raise ValidationError("field's size element must be an integer")

            total_size = value.get('total_size', None)
            if total_size is not None and not isinstance(total_size, int):
                raise ValidationError("field's total_size element must be an integer")

            refs = value.get('refs', None)
            if refs is not None and not isinstance(refs, list):
                # TODO: Validate that all refs are strings.
                raise ValidationError("field's refs element must be a list of strings")

            return DirDescriptor(
                path,
                size=size,
                total_size=total_size,
                refs=refs,
            )
        elif not isinstance(value, None):
            raise ValidationError("field must be a DirDescriptor, string or a dict")
706055
def from_list(cls, database, key, data, clear=False):
        
        arr = cls(database, key)
        if clear:
            arr.clear()
        arr.extend(data)
        return arr
927024
def record_to_objects(self):
        """"""
        from ambry.orm import Column, Table, Dataset

        def _clean_int(i):
            if i is None:
                return None
            elif isinstance(i, int):
                return i
            elif isinstance(i, string_types):
                if len(i) == 0:
                    return None

                return int(i.strip())

        bsfile = self.record

        contents = bsfile.unpacked_contents

        if not contents:
            return

        line_no = 1  # Accounts for file header. Data starts on line 2

        errors = []
        warnings = []

        extant_tables = {t.name: t for t in self._dataset.tables}

        old_types_map = {
            'varchar': Column.DATATYPE_STR,
            'integer': Column.DATATYPE_INTEGER,
            'real': Column.DATATYPE_FLOAT,
        }

        def run_progress_f(line_no):
            self._bundle.log('Loading tables from file. Line #{}'.format(line_no))

        from ambry.bundle.process import CallInterval
        run_progress_f = CallInterval(run_progress_f, 10)

        table_number = self._dataset._database.next_sequence_id(Dataset, self._dataset.vid, Table)
        for row in bsfile.dict_row_reader:

            line_no += 1

            run_progress_f(line_no)

            # Skip blank lines
            if not row.get('column', False) and not row.get('table', False):
                continue

            if not row.get('column', False):
                raise ConfigurationError('Row error: no column on line {}'.format(line_no))

            if not row.get('table', False):
                raise ConfigurationError('Row error: no table on line {}'.format(line_no))

            if not row.get('datatype', False) and not row.get('valuetype', False):
                raise ConfigurationError('Row error: no type on line {}'.format(line_no))

            value_type = row.get('valuetype', '').strip() if row.get('valuetype', False) else None
            data_type = row.get('datatype', '').strip() if row.get('datatype', False) else None

            def resolve_data_type(value_type):
                from ambry.valuetype import resolve_value_type
                vt_class = resolve_value_type(value_type)

                if not vt_class:
                    raise ConfigurationError("Row error: unknown valuetype '{}'".format(value_type))

                return vt_class.python_type().__name__

            # If we have a value type field, and not the datatype,
            # the value type is as specified, and the data type is derived from it.
            if value_type and not data_type:
                data_type = resolve_data_type(value_type)

            elif data_type and not value_type:
                value_type = data_type
                data_type = resolve_data_type(value_type)

            # There are still some old data types hanging around
            data_type = old_types_map.get(data_type.lower(), data_type)

            table_name = row['table']

            try:
                table = extant_tables[table_name]
            except KeyError:
                table = self._dataset.new_table(
                    table_name,
                    sequence_id=table_number,
                    description=row.get('description') if row['column'] == 'id' else ''
                )

                table_number += 1
                extant_tables[table_name] = table

            data = {k.replace('d_', '', 1): v
                    for k, v in list(row.items()) if k and k.startswith('d_') and v}

            if row['column'] == 'id':
                table.data.update(data)
                data = {}

            table.add_column(
                row['column'],
                fk_vid=row['is_fk'] if row.get('is_fk', False) else None,
                description=(row.get('description', '') or '').strip(),
                datatype=data_type,
                valuetype=value_type,
                parent=row.get('parent'),
                proto_vid=row.get('proto_vid'),
                size=_clean_int(row.get('size', None)),
                width=_clean_int(row.get('width', None)),
                data=data,
                keywords=row.get('keywords'),
                measure=row.get('measure'),
                transform=row.get('transform'),
                derivedfrom=row.get('derivedfrom'),
                units=row.get('units', None),
                universe=row.get('universe'),
                update_existing= True)

        self._dataset.t_sequence_id = table_number

        return warnings, errors
412466
def _finalize_db(self):
        """"""
        for k, v in self._els_idxed.items():
            self._els_idxed[k] = dict(v)
        self._els_idxed = dict(self._els_idxed)
139459
def to_python(self, value):
        """"""
        cache = None
        if value is None:
            return None

        if isinstance(value, DataDescriptor):
            return value
        elif isinstance(value, dict):
            # Allow pre-hydrated data objects.
            cache = value
            try:
                value = cache['__id']
            except KeyError:
                raise ValidationError("dictionary must contain an '__id' element")
        elif not isinstance(value, int):
            raise ValidationError("field must be a DataDescriptor, data object primary key or dict")

        return DataDescriptor(value, self, cache)
1356141
def migrate(engine, database, module, **kwargs):
    '''
    
    '''
    validate_args(engine, database, module)

    options = {
        'direction': kwargs.get('direction', 'up'),
        'fake': kwargs.get('fake', False),
        'force': kwargs.get('force', False),
        'migration': kwargs.get('migration', None),
        'transaction': kwargs.get('transaction', True),
    }

    Migration._meta.database = database
    migrator = DATABASE_MAP[engine](database, module, **options)
    migrator.run()
142305
def convert(obj, ids, attr_type, item_func, cdata, parent='root'):
    """"""
    
    LOG.info('Inside convert(). obj type is: "%s", obj="%s"' % (type(obj).__name__, unicode_me(obj)))
    
    item_name = item_func(parent)
    
    if isinstance(obj, numbers.Number) or type(obj) in (str, unicode):
        return convert_kv(item_name, obj, attr_type, cdata)
        
    if hasattr(obj, 'isoformat'):
        return convert_kv(item_name, obj.isoformat(), attr_type, cdata)
        
    if type(obj) == bool:
        return convert_bool(item_name, obj, attr_type, cdata)
        
    if obj is None:
        return convert_none(item_name, '', attr_type, cdata)
        
    if isinstance(obj, dict):
        return convert_dict(obj, ids, parent, attr_type, item_func, cdata)
        
    if isinstance(obj, collections.Iterable):
        return convert_list(obj, ids, parent, attr_type, item_func, cdata)
        
    raise TypeError('Unsupported data type: %s (%s)' % (obj, type(obj).__name__))
479579
def db_import(ctx, schema, uuid, object_filter, import_format, filename, all_schemata, dry):
    """"""

    import_format = import_format.upper()

    with open(filename, 'r') as f:
        json_data = f.read()
    data = json.loads(json_data)  # , parse_float=True, parse_int=True)

    if schema is None:
        if all_schemata is False:
            log('No schema given. Read the help', lvl=warn)
            return
        else:
            schemata = data.keys()
    else:
        schemata = [schema]

    from hfos import database
    database.initialize(ctx.obj['dbhost'], ctx.obj['dbname'])

    all_items = {}
    total = 0

    for schema_item in schemata:
        model = database.objectmodels[schema_item]

        objects = data[schema_item]
        if uuid:
            for item in objects:
                if item['uuid'] == uuid:
                    items = [model(item)]
        else:
            items = []
            for item in objects:
                thing = model(item)
                items.append(thing)

        schema_total = len(items)
        total += schema_total

        if dry:
            log('Would import', schema_total, 'items of', schema_item)
        all_items[schema_item] = items

    if dry:
        log('Would import', total, 'objects.')
    else:
        log('Importing', total, 'objects.')
        for schema_name, item_list in all_items.items():
            log('Importing', len(item_list), 'objects of type', schema_name)
            for item in item_list:
                item._fields['_id'] = bson.objectid.ObjectId(item._fields['_id'])
                item.save()
1188444
def to_sqlite3(self, location=":memory:"):
        """"""

        def decimal_to_sqlite3(n):
            return int(100*n)

        def sqlite3_to_decimal(s):
            return Decimal(s)/100

        sqlite3.register_adapter(Decimal, decimal_to_sqlite3)
        sqlite3.register_converter("decimal", sqlite3_to_decimal)

        con = sqlite3.connect(location, detect_types=sqlite3.PARSE_COLNAMES |
                sqlite3.PARSE_DECLTYPES)
        cur = con.cursor()
        cur.execute("""create table transactions(
                           id primary key,
                           xfer date,
                           posted date,
                           message text,
                           amount decimal,
                           total decimal)""")
        for t in self:
            cur.execute("INSERT INTO transactions values(?,?,?,?,?,?)",
                (t.index, t.xfer, t.posted, t.message, t.amount, t.total))
        return con
404217
def _convert_entry(self, entry):
        """"""
        result = {}
        for key, value in entry.items():
            if isinstance(value, list):
                result[key] = [self._convert_field(key, val) for val in value]
            else:
                result[key] = self._convert_field(key, value)
        return result
794593
def to_python(self, value):
        """"""
        if isinstance(value, FileDescriptor):
            return value
        elif isinstance(value, str):
            return FileDescriptor(value)
        elif isinstance(value, dict):
            try:
                path = value['file']
            except KeyError:
                raise ValidationError("dictionary must contain a 'file' element")

            if not isinstance(path, str):
                raise ValidationError("field's file element must be a string")

            size = value.get('size', None)
            if size is not None and not isinstance(size, int):
                raise ValidationError("field's size element must be an integer")

            total_size = value.get('total_size', None)
            if total_size is not None and not isinstance(total_size, int):
                raise ValidationError("field's total_size element must be an integer")

            is_remote = value.get('is_remote', None)
            if is_remote is not None and not isinstance(is_remote, bool):
                raise ValidationError("field's is_remote element must be a boolean")

            file_temp = value.get('file_temp', None)
            if file_temp is not None and not isinstance(file_temp, str):
                raise ValidationError("field's file_temp element must be a string")

            refs = value.get('refs', None)
            if refs is not None and not isinstance(refs, list):
                # TODO: Validate that all refs are strings.
                raise ValidationError("field's refs element must be a list of strings")

            return FileDescriptor(
                path,
                size=size,
                total_size=total_size,
                is_remote=is_remote,
                file_temp=file_temp,
                refs=refs,
            )
        elif not isinstance(value, None):
            raise ValidationError("field must be a FileDescriptor, string or a dict")
922808
def convert(self, args, handler=None):
        """"""
        name = args
        field = attr = None
        opts = ()
        if isinstance(args, (list, tuple)):
            name, *opts = args
            if opts:
                attr = opts.pop()
            if opts:
                field = opts.pop()

        if not field and handler and handler.Schema:
            field = handler.Schema._declared_fields.get(attr or name) or \
                self.FILTER_CLASS.field_cls()
            field.attribute = field.attribute or attr or name
        return self.FILTER_CLASS(name, attr=attr, field=field, *opts)
610942
def to_python(self, value):
        """"""
        if value is None or value == '':
            return value
        if isinstance(value, base.Spec):
            return value
        return base.Spec(value)
332398
def to_python(self, value):
        
        if not value:
            value = []

        # If a list is set then validated its items
        if isinstance(value, list):
            py_list = self.validate_list(value)
        else:  # try to de-serialize value and expect list and then validate
            try:
                py_list = json.loads(value)

                if not isinstance(py_list, list):
                    raise TypeError

                self.validate_list(py_list)
            except (ValueError, TypeError):
                raise ValidationError(u'Must be a valid list of strings.')

        return py_list