loads the dataset and converts it to the correct format
350981
public function load($data, $dataType = 'json')
        {
            $this->_data = ($dataType != 'json') ? $this->dataToJson($data) : $data;
        }
1350120
def as_dataset(obj):
    
    if hasattr(obj, 'to_dataset'):
        obj = obj.to_dataset()
    if not isinstance(obj, Dataset):
        obj = Dataset(obj)
    return obj
358036
def _load(self, scale=1.0):
        

        LOG.debug("File: %s", str(self.requested_band_filename))
        ncf = Dataset(self.requested_band_filename, 'r')

        wvl = ncf.variables['wavelength'][:] * scale
        resp = ncf.variables['response'][:]

        self.rsr = {'wavelength': wvl, 'response': resp}
1168476
def _loadData(self, data):
        
        self._data = data
        self.codec = data.attrib.get('codec')
        self.codecID = data.attrib.get('codecID')
        self.id = cast(int, data.attrib.get('id'))
        self.index = cast(int, data.attrib.get('index', '-1'))
        self.language = data.attrib.get('language')
        self.languageCode = data.attrib.get('languageCode')
        self.selected = cast(bool, data.attrib.get('selected', '0'))
        self.streamType = cast(int, data.attrib.get('streamType'))
        self.type = cast(int, data.attrib.get('streamType'))
322473
def _load_dataset_info():
    
    def load_json(path):
        raw = pkgutil.get_data('vega_datasets', path)
        return json.loads(bytes_decode(raw))

    info = load_json('datasets.json')
    descriptions = load_json('dataset_info.json')
    local_datasets = load_json('local_datasets.json')

    for name in info:
        info[name]['is_local'] = (name in local_datasets)
    for name in descriptions:
        info[name].update(descriptions[name])

    return info
1167175
def load(app_id):
    """"""
    path_to_data = find_experiment_export(app_id)
    if path_to_data is None:
        raise IOError("Dataset {} could not be found.".format(app_id))

    return Data(path_to_data)
618105
def upload_dataset(self, dataset, dataset_name, schema_path=None):
        

        url = self.address + "/datasets/" + dataset_name + "/uploadSample"
        header = self.__check_authentication()

        fields = dict()
        remove = False
        if isinstance(dataset, GDataframe):
            tmp_path = TempFileManager.get_new_dataset_tmp_folder()
            dataset.to_dataset_files(local_path=tmp_path)
            dataset = tmp_path
            remove = True

        # a path is provided
        if not isinstance(dataset, str):
            raise TypeError("Dataset can be a path or a GDataframe. {} was passed".format(type(dataset)))

        file_paths, schema_path_found = Loader.get_file_paths(dataset)
        if schema_path is None:
            schema_path = schema_path_found
        fields['schema'] = (os.path.basename(schema_path), open(schema_path, "rb"), 'application/octet-stream')
        for i, file in enumerate(file_paths):
            fields["file"+str(i + 1)] = (os.path.basename(file), open(file, "rb"), 'application/octet-stream')

        encoder = MultipartEncoder(fields)
        callback = create_callback(encoder, len(fields))

        m_encoder = MultipartEncoderMonitor(encoder, callback)
        header['Content-Type'] = m_encoder.content_type

        self.logger.debug("Uploading dataset at {} with name {}".format(dataset, dataset_name))

        response = requests.post(url, data=m_encoder,
                                 headers=header)

        # closing files
        for fn in fields.keys():
            _, f, _ = fields[fn]
            f.close()

        if response.status_code != 200:
            raise ValueError("Code {}: {}".format(response.status_code, response.content))

        if remove:
            TempFileManager.delete_tmp_dataset(dataset)
1145162
def load(self, ds, verbose=False):
        
        try:
            self.fetchone('select ID from dataset')
        except sqlite3.OperationalError:
            self.create(force=True)
        self.unload(ds)
        dataset = ds.cldf.wl
        tables, ref_tables = schema(dataset)

        # update the DB schema:
        for t in tables:
            if self._create_table_if_not_exists(t):
                continue
            db_cols = self.tables[t.name]
            for col in t.columns:
                if col.name not in db_cols:
                    with self.connection() as conn:
                        conn.execute(
                            "ALTER TABLE {0} ADD COLUMN `{1.name}` {1.db_type}".format(
                                t.name, col))
                else:
                    if db_cols[col.name] != col.db_type:
                        raise ValueError(
                            'column {0}:{1} {2} redefined with new type {3}'.format(
                                t.name, col.name, db_cols[col.name], col.db_type))

        for t in ref_tables.values():
            self._create_table_if_not_exists(t)

        self.update_schema()

        # then load the data:
        with self.connection() as db:
            db.execute('PRAGMA foreign_keys = ON;')
            insert(
                db,
                'dataset',
                'ID,name,version,metadata_json',
                (
                    ds.id,
                    '{0}'.format(dataset),
                    git_hash(ds.dir),
                    dumps(dataset.metadata_dict)))
            insert(
                db,
                'datasetmeta',
                'dataset_ID,key,value',
                *[(ds.id, k, '{0}'.format(v)) for k, v in dataset.properties.items()])

            # load sources:
            rows = []
            for src in dataset.sources.items():
                values = [ds.id, src.id, src.genre] + [src.get(k) for k in BIBTEX_FIELDS]
                values.append(
                    dumps({k: v for k, v in src.items() if k not in BIBTEX_FIELDS}))
                rows.append(tuple(values))
            insert(
                db,
                'SourceTable',
                ['dataset_ID', 'ID', 'bibtex_type'] + BIBTEX_FIELDS + ['extra'],
                *rows)

            # For regular tables, we extract and keep references to sources.
            refs = defaultdict(list)

            for t in tables:
                # We want to lookup columns by the name used in the CLDF dataset.
                cols = {col.cldf_name: col for col in t.columns}
                # But we also want to look up primary keys by the database column name.
                cols_by_name = {col.name: col for col in t.columns}

                ref_table = ref_tables.get(t.name)
                rows, keys = [], []
                try:
                    for row in dataset[t.name]:
                        keys, values = ['dataset_ID'], [ds.id]
                        for k, v in row.items():
                            if ref_table and k == ref_table.consumes:
                                col = cols_by_name[t.primary_key]
                                refs[ref_table.name].append((row[col.cldf_name], v))
                            else:
                                col = cols[k]
                                if isinstance(v, list):
                                    v = (col.separator or ';').join(
                                        nfilter(col.convert(vv) for vv in v))
                                else:
                                    v = col.convert(v)
                                keys.append("`{0}`".format(col.name))
                                values.append(v)
                        keys, values = self.update_row(t.name, keys, values)
                        rows.append(tuple(values))
                    insert(db, t.name, keys, *rows, **{'verbose': verbose})
                except FileNotFoundError:
                    if t.name != 'CognateTable':  # An empty CognateTable is allowed.
                        raise  # pragma: no cover

            # Now insert the references, i.e. the associations with sources:
            for tname, items in refs.items():
                rows = []
                for oid, sources in items:
                    for source in sources:
                        sid, context = Sources.parse(source)
                        rows.append([ds.id, oid, sid, context])
                oid_col = '{0}_ID'.format(tname.replace('Source', ''))
                insert(db, tname, ['dataset_ID', oid_col, 'Source_ID', 'Context'], *rows)
            db.commit()
968934
def load_data(verbose=False):
    ""

    df = pd.read_csv(STOPS_PATH, usecols=['stop_id', 'stop_name'])
    if verbose:
        print('- Loaded %d entries from "%s".' % (len(df), STOPS_PATH))
    return df
23336
def load_mnist():
    ''''''
    mnist = skdata.mnist.dataset.MNIST()
    mnist.meta  # trigger download if needed.

    def arr(n, dtype):
        arr = mnist.arrays[n]
        return arr.reshape((len(arr), -1)).astype(dtype)
    train_images = arr('train_images', np.float32) / 128 - 1
    train_labels = arr('train_labels', np.uint8)
    return ((train_images[:50000], train_labels[:50000, 0]),
            (train_images[50000:], train_labels[50000:, 0]))
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
999902
def _load(self, dataset='train'):
        
        data, labels = None, None
        if dataset is 'train':
            files = [os.path.join(self.cifar10_dir, 'data_batch_%d' % i) for i in range(1, 6)]
        else:
            files = [os.path.join(self.cifar10_dir, 'test_batch')]

        for file in files:
            if not os.path.exists(file):
                raise FileNotFoundError('Failed to find file: ' + file)

        # Load the data from the batch files
        for file in files:
            with open(file, 'rb') as f:
                cifar10 = pickle.load(f, encoding='latin1')

            if labels is None:
                labels = np.array(cifar10['labels'])
            else:
                labels = np.concatenate((labels, cifar10['labels']), axis=0)

            if data is None:
                data = cifar10['data']
            else:
                data = np.concatenate((data, cifar10['data']), axis=0)

        # Adapt the format of the data to our convnet
        data = np.array(data, dtype=float) / 255.0
        data = data.reshape([-1, self.num_channels, self.img_size, self.img_size])
        data = data.transpose([0, 2, 3, 1])

        # One-hot encode labels (see https://stackoverflow.com/a/42874726)
        labels = np.eye(self.num_classes)[np.array(labels).reshape(-1)]

        if dataset is 'train':
            self._train_data, self._train_labels = data, labels
        else:
            self._test_data, self._test_labels = data, labels
731388
def _load_dataset_with_area(self, dsid, coords):
        """"""
        file_handlers = self._get_file_handlers(dsid)
        if not file_handlers:
            return

        area = self._load_dataset_area(dsid, file_handlers, coords)

        try:
            ds = self._load_dataset_data(file_handlers, dsid)
        except (KeyError, ValueError) as err:
            logger.exception("Could not load dataset '%s': %s", dsid, str(err))
            return None

        if area is not None:
            ds.attrs['area'] = area
            calc_coords = (('x' not in ds.coords) or('y' not in ds.coords)) and hasattr(area, 'get_proj_vectors_dask')
            if calc_coords and hasattr(area, 'get_proj_vectors'):
                ds['x'], ds['y'] = area.get_proj_vectors()
            elif calc_coords:
                # older pyresample with dask-only method
                ds['x'], ds['y'] = area.get_proj_vectors_dask(CHUNK_SIZE)
        return ds
742501
def get_dataset(self, key, info):
        """"""
        logger.debug('Reading dataset {}'.format(key.name))

        # Read data from file and calibrate if necessary
        if 'longitude' in key.name:
            data = self.nc['lon']
        elif 'latitude' in key.name:
            data = self.nc['lat']
        else:
            raise KeyError("Unknown dataset: {}".format(key.name))

        # Set proper dimension names
        data = data.rename({'xc': 'x', 'yc': 'y'})

        # Update metadata
        data.attrs.update(info)
        return data
1109652
def choose_raw_dataset(currently=""):
    """"""
    folder = os.path.join(get_project_root(), "raw-datasets")
    files = [os.path.join(folder, name) for name in os.listdir(folder)
             if name.endswith(".pickle")]
    default = -1
    for i, filename in enumerate(files):
        if os.path.basename(currently) == os.path.basename(filename):
            default = i
        if i != default:
            print("[%i]\t%s" % (i, os.path.basename(filename)))
        else:
            print("\033[1m[%i]\033[0m\t%s" % (i, os.path.basename(filename)))
    i = input_int_default("Choose a dataset by number: ", default)
    return files[i]
811911
def dataset_format(self):
        
        data = javabridge.call(self.jobject, "getDatasetFormat", "()Lweka/core/Instances;")
        if data is None:
            return None
        else:
            return Instances(data)
1256354
def save_datasets(self, writer="geotiff", datasets=None, compute=True, **kwargs):
        """"""
        if datasets is not None:
            datasets = [self[ds] for ds in datasets]
        else:
            datasets = [self.datasets.get(ds) for ds in self.wishlist]
            datasets = [ds for ds in datasets if ds is not None]
        if not datasets:
            raise RuntimeError("None of the requested datasets have been "
                               "generated or could not be loaded. Requested "
                               "composite inputs may need to have matching "
                               "dimensions (eg. through resampling).")
        writer, save_kwargs = load_writer(writer, ppp_config_dir=self.ppp_config_dir, **kwargs)
        return writer.save_datasets(datasets, compute=compute, **save_kwargs)