Creates named tuple with info needed to create a dataset.

        :return: named tuple
507872
def _prepare_data(self):
        
        self._extract_genes()
        self._extract_total_number_of_chars()
        self._extract_number_of_taxa()
        self._extract_reading_frames()

        Data = namedtuple('Data', ['gene_codes', 'number_taxa', 'number_chars',
                                   'seq_records', 'gene_codes_and_lengths',
                                   'reading_frames'])
        self.data = Data(self.gene_codes, self.number_taxa, self.number_chars,
                         self.seq_records, self._gene_codes_and_lengths,
                         self.reading_frames)
274599
def named_tuple(
        element_name,  # type: Text
        tuple_type,  # type: Type[Tuple]
        child_processors,  # type: List[Processor]
        required=True,  # type: bool
        alias=None,  # type: Optional[Text]
        hooks=None  # type: Optional[Hooks]
):
    # type: (...) -> RootProcessor
    
    converter = _named_tuple_converter(tuple_type)
    processor = _Aggregate(element_name, converter, child_processors, required, alias)
    return _processor_wrap_if_hooks(processor, hooks)
1192129
def to_named_tuple(obj, visited_children=None, back_relationships=None, levels=None, ignore=[], extras={}):
    

    if visited_children is None:
        visited_children = []

    if back_relationships is None:
        back_relationships = []

    serialized_data = {c.key: getattr(obj, c.key) for c in obj.__table__.columns}


    #Any other non-column data to include in the keyed tuple
    for k, v in extras.items():
        serialized_data[k] = v

    relationships = class_mapper(obj.__class__).relationships

    #Set the attributes to 'None' first, so the attributes are there, even if they don't
    #get filled in:
    for name, relation in relationships.items():
        if relation.uselist:
            serialized_data[name] = tuple([])
        else:
            serialized_data[name] = None


    visitable_relationships = [(name, rel) for name, rel in relationships.items() if name not in back_relationships]

    if levels is not None and levels > 0:
        for name, relation in visitable_relationships:

            levels = levels - 1

            if name in ignore:
                continue

            if relation.backref:
                back_relationships.append(relation.backref)

            relationship_children = getattr(obj, name)

            if relationship_children is not None:
                if relation.uselist:
                    children = []
                    for child in [c for c in relationship_children if c not in visited_children]:
                        visited_children.append(child)
                        children.append(to_named_tuple(child, visited_children, back_relationships, ignore=ignore, levels=levels))
                    serialized_data[name] = tuple(children)
                else:
                    serialized_data[name] = to_named_tuple(relationship_children, visited_children, back_relationships, ignore=ignore, levels=levels)

    vals = []
    cols = []
    for k, v in serialized_data.items():
        vals.append(k)
        cols.append(v)

    result = KeyedTuple(cols, vals)

    return result
825809
def create_dataset(self, group, chunk_size):
        """"""
        group.attrs['format'] = self.dformat
        group.attrs['dim'] = self.dim

        if chunk_size == 'auto':
            group.create_dataset(
                'coordinates', (0, 2), dtype=np.float64,
                chunks=True, maxshape=(None, 2))

            group.create_dataset(
                self.name, (0,), dtype=self.dtype,
                chunks=True, maxshape=(None,))

        else:
            # for storing sparse data we don't use the self.nb_per_chunk,
            # which is only used by the Writer to determine times chunking.
            per_chunk = nb_per_chunk(self.dtype.itemsize, 1, chunk_size)

            group.create_dataset(
                'coordinates', (0, 2), dtype=np.float64,
                chunks=(per_chunk, 2), maxshape=(None, 2))

            group.create_dataset(
                self.name, (0,), dtype=self.dtype,
                chunks=(per_chunk,), maxshape=(None,))

        dtype = np.int64
        if chunk_size == 'auto':
            chunks = True
            self.nb_per_chunk = 'auto'
        else:
            chunks = (nb_per_chunk(np.dtype(dtype).itemsize, 1, chunk_size),)
            # Needed by Times.create_dataset
            self.nb_per_chunk = nb_per_chunk(
                self.dtype.itemsize,
                int(round(self.sparsity*self.dim)),
                chunk_size)

        group.create_dataset(
            'frames', (0,), dtype=dtype,
            chunks=chunks, maxshape=(None,))
1263417
def to_named_tuple(keys, values):
    

    values = [dbobject.__dict__[key] for key in dbobject.keys()]

    tuple_object = namedtuple('DBObject', dbobject.keys())

    tuple_instance = tuple_object._make(values)

    return tuple_instance
292706
def new_dataset(args):
    
    if not args.args:
        raise ParserError('you must specify an existing directory')
    outdir = Path(args.args.pop(0))
    if not outdir.exists():
        raise ParserError('you must specify an existing directory')

    id_pattern = re.compile('[a-z_0-9]+$')
    md = {}
    if args.args:
        md['id'] = args.args.pop(0)
    else:
        md['id'] = input('Dataset ID: ')

    while not id_pattern.match(md['id']):
        print('dataset id must only consist of lowercase ascii letters, digits and _ (underscore)!')
        md['id'] = input('Dataset ID: ')

    outdir = outdir / md['id']
    if not outdir.exists():
        outdir.mkdir()

    for key in ['title', 'url', 'license', 'conceptlist', 'citation']:
        md[key] = input('Dataset {0}: '.format(key))

    # check license!
    # check conceptlist!

    for path in Path(pylexibank.__file__).parent.joinpath('dataset_template').iterdir():
        if path.is_file():
            if path.suffix in ['.pyc']:
                continue  # pragma: no cover
            target = path.name
            content = read_text(path)
            if '+' in path.name:
                target = re.sub(
                    '\+([a-z]+)\+',
                    lambda m: '{' + m.groups()[0] + '}',
                    path.name
                ).format(**md)
            if target.endswith('_tmpl'):
                target = target[:-5]
                content = content.format(**md)
            write_text(outdir / target, content)
        else:
            target = outdir / path.name
            if target.exists():
                shutil.rmtree(str(target))
            shutil.copytree(str(path), str(target))
    del md['id']
    jsonlib.dump(md, outdir / 'metadata.json', indent=4)
1262094
def dataset(self):
        ''''''

        # Initialize the addcorpus subcommand's argparser
        description = '''The dataset subcommand has a number of subcommands of its own, including:
            list\t-\tlists all available datasets in dsrt's library
            prepare\t-\tprocesses a corpus into a dataset and adds the processed dataset to dsrt's library'''

        parser = argparse.ArgumentParser(description=description)
        self.init_dataset_args(parser)

        # parse the args we got
        args = parser.parse_args(sys.argv[2:3])

        corpus_command = 'dataset_' + args.dataset_command

        if not hasattr(self, corpus_command):
            print('Unrecognized dataset command.')
            parser.print_help()
            exit(1)

        getattr(self, corpus_command)()
404613
def dataset(self):
        
        if self.dataset_format:
            return ".".join([self.dataset_name, self.dataset_format])
        return self.dataset_name
979047
def create_dataset(
            self, group, chunk_size, compression=None, compression_opts=None):
        """"""
        group.attrs['format'] = self.dformat
        super(Features, self)._create_dataset(
            group, chunk_size, compression, compression_opts)

        # TODO attribute declared outside __init__ is not safe. Used
        # because Labels.create_dataset need it.
        if chunk_size != 'auto':
            self.nb_per_chunk = nb_per_chunk(
                self.dtype.itemsize, self.dim, chunk_size)
        else:
            self.nb_per_chunk = 'auto'
1207708
def create_image_list(self, dataset, fns_idxs):
        ""
        items = dataset.x.items
        if self._duplicates:
            chunked_idxs = chunks(fns_idxs, 2)
            chunked_idxs = [chunk for chunk in chunked_idxs if Path(items[chunk[0]]).is_file() and Path(items[chunk[1]]).is_file()]
            return  [(dataset.x[i]._repr_jpeg_(), items[i], self._labels[dataset.y[i].data]) for chunk in chunked_idxs for i in chunk]
        else:
            return [(dataset.x[i]._repr_jpeg_(), items[i], self._labels[dataset.y[i].data]) for i in fns_idxs if
                    Path(items[i]).is_file()]
1223231
def generate_dataset_shelve(filename, number_items=1000):
    
    if os.path.exists(filename):
        os.remove(filename)
    data = shelve.open(filename)
    names = get_names()
    totalnames = len(names)
    #init random seeder
    random.seed()
    #calculate items
#    names = random.sample(names, number_items)
    for i in range(number_items):
        data[str(i+1)] = {"name":names[random.randint(0,totalnames-1)],
                     "age":random.randint(1,100),
                     "description":li_words(50, False)}
    data.close()
1000469
def tuple(self):
        
        return tuple(getattr(self, k) for k in self.__class__.defaults)
1316746
def data_available(dataset_name=None):
    """"""
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    dr = data_resources[dataset_name]
    zip_urls = (dr['files'], )
    if 'save_names' in dr: zip_urls += (dr['save_names'], )
    else: zip_urls += ([],)

    for file_list, save_list in zip_longest(*zip_urls, fillvalue=[]):
        for f, s in zip_longest(file_list, save_list, fillvalue=None):
            if s is not None: f=s # If there is a save_name given, use that one
            if not os.path.exists(os.path.join(data_path, dataset_name, f)):
                return False
    return True
813654
def as_named_tuple(self):
        
        keys = list(self.entities.keys())
        replaced = []
        for i, k in enumerate(keys):
            if iskeyword(k):
                replaced.append(k)
                keys[i] = '%s_' % k
        if replaced:
            safe = ['%s_' % k for k in replaced]
            warnings.warn("Entity names cannot be reserved keywords when "
                          "representing a File as a namedtuple. Replacing "
                          "entities %s with safe versions %s." % (keys, safe))
        entities = dict(zip(keys, self.entities.values()))
        _File = namedtuple('File', 'filename ' + ' '.join(entities.keys()))
        return _File(filename=self.path, **entities)
498697
def group_attrib(self):
        '''
        
        '''
        group_attributes = [g.attrib for g in self.dataset.groups if self in g]
        if group_attributes:
            return concat_namedtuples(*group_attributes)
807210
def _make_dataset(cls, coords):
        
        class Slice(cls._SliceType):
            extra_coords = coords

        Slice.__name__ = '%s.slice(%s)' % (
            cls.__name__,
            ', '.join('%s=%r' % item for item in coords.items()),
        )
        return Slice
1184293
def as_namedtuple(self):
        
        d = self.as_dict()
        return namedtuple('ColorRegister', d.keys())(*d.values())