Create a dataset as quickly as possible.

    Beware: the `variables` OrderedDict is modified INPLACE.
350345
def _fast_dataset(
    variables: 'OrderedDict[Any, Variable]',
    coord_variables: Mapping[Any, Variable],
) -> 'Dataset':
    
    from .dataset import Dataset
    variables.update(coord_variables)
    coord_names = set(coord_variables)
    return Dataset._from_vars_and_coord_names(variables, coord_names)
858598
def create(cls, train_ds:Dataset, valid_ds:Dataset, test_ds:Optional[Dataset]=None, path:PathOrStr='.', bs:int=64,
               val_bs:int=None, num_workers:int=defaults.cpus, dl_tfms:Optional[Collection[Callable]]=None,
               device:torch.device=None, collate_fn:Callable=data_collate, no_check:bool=False, **dl_kwargs)->'DataBunch':
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        dls = [DataLoader(d, b, shuffle=s, drop_last=s, num_workers=num_workers, **dl_kwargs) for d,b,s in
               zip(datasets, (bs,val_bs,val_bs,val_bs), (True,False,False,False)) if d is not None]
        return cls(*dls, path=path, device=device, dl_tfms=dl_tfms, collate_fn=collate_fn, no_check=no_check)
390017
def input_dataset_from_dataframe(df, delays=(1, 2, 3), inputs=(1, 2, -1), outputs=None, normalize=True, verbosity=1):
    
    return dataset_from_dataframe(df=df, delays=delays, inputs=inputs, outputs=outputs,
                                  normalize=normalize, verbosity=verbosity)
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs
558703
def _create_dataset(
            self, group, chunk_size, compression, compression_opts):
        """"""
        if chunk_size == 'auto':
            chunks = True
        else:
            # if dtype is a variable str, guess representative size is 20 bytes
            per_chunk = (
                nb_per_chunk(20, 1, chunk_size) if self.dtype == np.dtype('O')
                else nb_per_chunk(
                        np.dtype(self.dtype).itemsize, 1, chunk_size))
            chunks = (per_chunk,)

        shape = (0,)
        maxshape = (None,)

        # raise if per_chunk >= 4 Gb, this is requested by h5py
        group.create_dataset(
            self.name, shape, dtype=self.dtype,
            chunks=chunks, maxshape=maxshape, compression=compression,
            compression_opts=compression_opts)
851998
def processDatasetBlocks(self, url, conn, inputdataset, order_counter):
        
        ordered_dict = {}
        srcblks = self.getSrcBlocks(url, dataset=inputdataset)
        if len(srcblks) < 0:
            e = "DBSMigration: No blocks in the required dataset %s found at source %s."%(inputdataset, url)
            dbsExceptionHandler('dbsException-invalid-input2', e, self.logger.exception, e)
        dstblks = self.blocklist.execute(conn, dataset=inputdataset)
	self.logger.debug("******* dstblks for dataset %s ***********" %inputdataset)
	self.logger.debug(dstblks)
        blocksInSrcNames = [ y['block_name'] for y in srcblks]
	blocksInDstNames = []
	for item in dstblks:
	    blocksInDstNames.append(item['block_name'])
        ordered_dict[order_counter] = []
        for ablk in blocksInSrcNames:
            if not ablk in blocksInDstNames:
                ordered_dict[order_counter].append(ablk)
        if ordered_dict[order_counter] != []:
	    self.logger.debug("**** ordered_dict dict length ****")
	    self.logger.debug(len(ordered_dict)) 	
            return ordered_dict
        else:
            return {}
807210
def _make_dataset(cls, coords):
        
        class Slice(cls._SliceType):
            extra_coords = coords

        Slice.__name__ = '%s.slice(%s)' % (
            cls.__name__,
            ', '.join('%s=%r' % item for item in coords.items()),
        )
        return Slice
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
513259
def _create_raw_data(self):
        
        result = {}
        for section in self.get_sections():
            result[section.get('id')] = 'all'
        return result
1284217
def create_model(cls, data: dict, fields=None):
        '''
        
        '''
        if fields is None:
            fields = set(cls._fields.keys())
        else:
            if not isinstance(fields, set):
                fields = set(fields)
        new_keys = set(data.keys()) - fields
        if new_keys:
            for new_key in new_keys:
                del data[new_key]
        return cls(data)
786683
def dataset_merge_method(dataset, other, overwrite_vars, compat, join):
    """"""

    # we are locked into supporting overwrite_vars for the Dataset.merge
    # method due for backwards compatibility
    # TODO: consider deprecating it?

    if isinstance(overwrite_vars, str):
        overwrite_vars = set([overwrite_vars])
    overwrite_vars = set(overwrite_vars)

    if not overwrite_vars:
        objs = [dataset, other]
        priority_arg = None
    elif overwrite_vars == set(other):
        objs = [dataset, other]
        priority_arg = 1
    else:
        other_overwrite = OrderedDict()
        other_no_overwrite = OrderedDict()
        for k, v in other.items():
            if k in overwrite_vars:
                other_overwrite[k] = v
            else:
                other_no_overwrite[k] = v
        objs = [dataset, other_no_overwrite, other_overwrite]
        priority_arg = 2

    return merge_core(objs, compat, join, priority_arg=priority_arg)
1130509
def push(description, datasets, wait, verbose):
    """"""
    # Loading config
    config = _get_config()
    file_patterns = config["contents"]
    if not isinstance(file_patterns, type([])):
        # put it into an array so we can iterate it, if it isn't already an array
        file_patterns = [file_patterns]
    if datasets:
        datasets_string = datasets.split(',')
    else:
        datasets_string = config.get("datasets", '')


    # Getting all file names/globs -- making sure we get CONFIG_FILE
    files = {CONFIG_FILE}  # use a set so we don't get duplicates
    for pattern in file_patterns:
        files.update(glob.glob(pattern))
    if not files:
        print "No files could be found? Check your contents section in `CONFIG_FILE`!"
        exit(-1)

    if verbose:
        print ""
        print "gatherered files:"
        for path in files:
            print "\t", path
        print ""

    # Zipping
    _print("zipping...")

    if not os.path.exists("brains_history"):
        os.mkdir("brains_history")

    zip_path = 'brains_history/%s.zip' % str(datetime.datetime.now())
    with ZipFile(zip_path, 'w') as zip_file:
        for path in files:
            zip_file.write(path)
    cprint("done", 'green')

    # Sending to server
    with open(zip_path, 'rb') as zip_file:
        _print("sending to server...")
        try:
            response = requests.post(
                URL_SUBMIT,
                files={
                    "zip_file": zip_file,
                },
                data={
                    "name": config["name"],
                    "description": description or '',
                    "languages": config["languages"],
                    "datasets": datasets_string,
                    "wait": wait,
                },
                stream=wait  # if we're waiting for response then stream
            )

            if response.status_code == 200:
                cprint("done", 'green')

                if wait:
                    _print("\nOutput: ")
                    cprint(" " * 72, 'green', attrs=('underline',))

                    chunk_buffer = ""
                    # read in 1 chunk at a time for carriage return detection
                    for chunk in response.iter_content(chunk_size=1):
                        chunk_buffer += chunk

                        if chunk == '\r':
                            # We hit the end of a message!
                            try:
                                data = json.loads(chunk_buffer)
                                if "stdout" not in data or "stderr" not in data:
                                    print "dis one"
                                    continue
                            except (ValueError,):
                                continue

                            if data["stdout"]:
                                # Get rid of termination string, if it's there
                                data["stdout"] = data["stdout"].replace("-%-%-%-%-END BRAIN SEQUENCE-%-%-%-%-", "")
                                _print(data["stdout"])
                            if data["stderr"]:
                                _print(colored(data["stderr"], 'red'))

                            # Clear buffer after reading message
                            chunk_buffer = ""
            else:
                cprint(response.json()["error"], 'red')
        except requests.exceptions.ConnectionError:
            cprint("failed to connect to server!", 'red')
            exit(-2)
1273187
def _dataset_concat(datasets, dim, data_vars, coords, compat, positions):
    
    from .dataset import Dataset

    if compat not in ['equals', 'identical']:
        raise ValueError("compat=%r invalid: must be 'equals' "
                         "or 'identical'" % compat)

    dim, coord = _calc_concat_dim_coord(dim)
    # Make sure we're working on a copy (we'll be loading variables)
    datasets = [ds.copy() for ds in datasets]
    datasets = align(*datasets, join='outer', copy=False, exclude=[dim])

    concat_over, equals = _calc_concat_over(datasets, dim, data_vars, coords)

    def insert_result_variable(k, v):
        assert isinstance(v, Variable)
        if k in datasets[0].coords:
            result_coord_names.add(k)
        result_vars[k] = v

    # create the new dataset and add constant variables
    result_vars = OrderedDict()
    result_coord_names = set(datasets[0].coords)
    result_attrs = datasets[0].attrs
    result_encoding = datasets[0].encoding

    for k, v in datasets[0].variables.items():
        if k not in concat_over:
            insert_result_variable(k, v)

    # check that global attributes and non-concatenated variables are fixed
    # across all datasets
    for ds in datasets[1:]:
        if (compat == 'identical' and
                not utils.dict_equiv(ds.attrs, result_attrs)):
            raise ValueError('dataset global attributes not equal')
        for k, v in ds.variables.items():
            if k not in result_vars and k not in concat_over:
                raise ValueError('encountered unexpected variable %r' % k)
            elif (k in result_coord_names) != (k in ds.coords):
                raise ValueError('%r is a coordinate in some datasets but not '
                                 'others' % k)
            elif k in result_vars and k != dim:
                # Don't use Variable.identical as it internally invokes
                # Variable.equals, and we may already know the answer
                if compat == 'identical' and not utils.dict_equiv(
                        v.attrs, result_vars[k].attrs):
                    raise ValueError(
                        'variable %s not identical across datasets' % k)

                # Proceed with equals()
                try:
                    # May be populated when using the "different" method
                    is_equal = equals[k]
                except KeyError:
                    result_vars[k].load()
                    is_equal = v.equals(result_vars[k])
                if not is_equal:
                    raise ValueError(
                        'variable %s not equal across datasets' % k)

    # we've already verified everything is consistent; now, calculate
    # shared dimension sizes so we can expand the necessary variables
    dim_lengths = [ds.dims.get(dim, 1) for ds in datasets]
    non_concat_dims = {}
    for ds in datasets:
        non_concat_dims.update(ds.dims)
    non_concat_dims.pop(dim, None)

    def ensure_common_dims(vars):
        # ensure each variable with the given name shares the same
        # dimensions and the same shape for all of them except along the
        # concat dimension
        common_dims = tuple(pd.unique([d for v in vars for d in v.dims]))
        if dim not in common_dims:
            common_dims = (dim,) + common_dims
        for var, dim_len in zip(vars, dim_lengths):
            if var.dims != common_dims:
                common_shape = tuple(non_concat_dims.get(d, dim_len)
                                     for d in common_dims)
                var = var.set_dims(common_dims, common_shape)
            yield var

    # stack up each variable to fill-out the dataset (in order)
    for k in datasets[0].variables:
        if k in concat_over:
            vars = ensure_common_dims([ds.variables[k] for ds in datasets])
            combined = concat_vars(vars, dim, positions)
            insert_result_variable(k, combined)

    result = Dataset(result_vars, attrs=result_attrs)
    result = result.set_coords(result_coord_names)
    result.encoding = result_encoding

    if coord is not None:
        # add concat dimension last to ensure that its in the final Dataset
        result[coord.name] = coord

    return result
72579
def _load_ancillary_variables(self, datasets):
        """"""
        all_av_ids = set()
        for dataset in datasets.values():
            ancillary_variables = dataset.attrs.get('ancillary_variables', [])
            if not isinstance(ancillary_variables, (list, tuple, set)):
                ancillary_variables = ancillary_variables.split(' ')
            av_ids = []
            for key in ancillary_variables:
                try:
                    av_ids.append(self.get_dataset_key(key))
                except KeyError:
                    logger.warning("Can't load ancillary dataset %s", str(key))

            all_av_ids |= set(av_ids)
            dataset.attrs['ancillary_variables'] = av_ids
        loadable_av_ids = [av_id for av_id in all_av_ids if av_id not in datasets]
        if not all_av_ids:
            return
        if loadable_av_ids:
            self.load(loadable_av_ids, previous_datasets=datasets)

        for dataset in datasets.values():
            new_vars = []
            for av_id in dataset.attrs.get('ancillary_variables', []):
                if isinstance(av_id, DatasetID):
                    new_vars.append(datasets[av_id])
                else:
                    new_vars.append(av_id)
            dataset.attrs['ancillary_variables'] = new_vars
1300298
def _auto_combine(datasets, concat_dims, compat, data_vars, coords,
                  infer_order_from_coords, ids):
    

    # Arrange datasets for concatenation
    if infer_order_from_coords:
        raise NotImplementedError
        # TODO Use coordinates to determine tile_ID for each dataset in N-D
        # Ignore how they were ordered previously
        # Should look like:
        # combined_ids, concat_dims = _infer_tile_ids_from_coords(datasets,
        # concat_dims)
    else:
        # Use information from the shape of the user input
        if not ids:
            # Determine tile_IDs by structure of input in N-D
            # (i.e. ordering in list-of-lists)
            combined_ids, concat_dims = _infer_concat_order_from_positions(
                datasets, concat_dims)
        else:
            # Already sorted so just use the ids already passed
            combined_ids = OrderedDict(zip(ids, datasets))

    # Check that the inferred shape is combinable
    _check_shape_tile_ids(combined_ids)

    # Repeatedly concatenate then merge along each dimension
    combined = _combine_nd(combined_ids, concat_dims, compat=compat,
                           data_vars=data_vars, coords=coords)
    return combined
1083124
def from_dict(cls, d, **kwargs):
        """"""
        args = []
        for k in DATASET_KEYS:
            val = kwargs.get(k, d.get(k))
            # force modifiers to tuple
            if k == 'modifiers' and val is not None:
                val = tuple(val)
            args.append(val)

        return cls(*args)