Given datasets in JSON format, build and return a list of dataset objects
769436
def build_datasets(datasets_json):
    
    if not datasets_json:
        return []

    datasets = []
    dataset_type_map = OrderedDict([
        ('generated', 'datasets'),
        ('used', 'prev_published_datasets')
    ])
    dataset_type_map_found = []
    # First look for the types of datasets present
    for dataset_key, dataset_type in iteritems(dataset_type_map):
        if datasets_json.get(dataset_key):
            dataset_type_map_found.append(dataset_key)
    # Continue with the found dataset types
    for dataset_key in dataset_type_map_found:
        dataset_type = dataset_type_map.get(dataset_key)
        for dataset_values in datasets_json.get(dataset_key):
            dataset = ea.Dataset()
            utils.set_attr_if_value(dataset, 'dataset_type', dataset_type)
            utils.set_attr_if_value(dataset, 'year', dataset_values.get('date'))
            utils.set_attr_if_value(dataset, 'title', dataset_values.get('title'))
            utils.set_attr_if_value(dataset, 'comment', dataset_values.get('details'))
            utils.set_attr_if_value(dataset, 'doi', dataset_values.get('doi'))
            utils.set_attr_if_value(dataset, 'uri', dataset_values.get('uri'))
            utils.set_attr_if_value(dataset, 'accession_id', dataset_values.get('dataId'))
            utils.set_attr_if_value(dataset, 'assigning_authority', 
                                    dataset_values.get('assigningAuthority'))
            # authors
            if dataset_values.get('authors'):
                # parse JSON format authors into author objects
                for author_json in dataset_values.get('authors'):
                    if utils.author_name_from_json(author_json):
                        dataset.add_author(utils.author_name_from_json(author_json))
            # Try to populate the doi attribute if the uri is a doi
            if not dataset.doi and dataset.uri:
                if dataset.uri != eautils.doi_uri_to_doi(dataset.uri):
                    dataset.doi = eautils.doi_uri_to_doi(dataset.uri)
            datasets.append(dataset)
    return datasets
447958
def _get_datasets(dataset_ids):
    

    dataset_dict = {}

    datasets = []
    if len(dataset_ids) > qry_in_threshold:
        idx = 0
        extent =qry_in_threshold
        while idx < len(dataset_ids):
            log.info("Querying %s datasets", len(dataset_ids[idx:extent]))
            rs = db.DBSession.query(Dataset).filter(Dataset.id.in_(dataset_ids[idx:extent])).all()
            datasets.extend(rs)
            idx = idx + qry_in_threshold

            if idx + qry_in_threshold > len(dataset_ids):
                extent = len(dataset_ids)
            else:
                extent = extent + qry_in_threshold
    else:
        datasets = db.DBSession.query(Dataset).filter(Dataset.id.in_(dataset_ids))


    for r in datasets:
        dataset_dict[r.id] = r

    log.info("Retrieved %s datasets", len(dataset_dict))

    return dataset_dict
159741
def __parseDatasets(self):
        """"""
        datasets = []
        if self.__dataItem.has_key('dataSets'):
            for dataset in self.__dataItem['dataSets']:
                datasets.append(DataSet(Sitools2Abstract.getBaseUrl(self), dataset))
        return datasets
59896
def get_datasets(dataset_ids,**kwargs):
    

    user_id = int(kwargs.get('user_id'))
    datasets = []
    if len(dataset_ids) == 0:
        return []
    try:
        dataset_rs = db.DBSession.query(Dataset.id,
                Dataset.type,
                Dataset.unit_id,
                Dataset.name,
                Dataset.hidden,
                Dataset.cr_date,
                Dataset.created_by,
                DatasetOwner.user_id,
                null().label('metadata'),
                case([(and_(Dataset.hidden=='Y', DatasetOwner.user_id is not None), None)],
                        else_=Dataset.value).label('value')).filter(
                Dataset.id.in_(dataset_ids)).outerjoin(DatasetOwner,
                                    and_(DatasetOwner.dataset_id==Dataset.id,
                                    DatasetOwner.user_id==user_id)).all()

        #convert the value row into a string as it is returned as a binary
        for dataset_row in dataset_rs:
            dataset_dict = dataset_row._asdict()

            if dataset_row.value is not None:
                dataset_dict['value'] = str(dataset_row.value)

            if dataset_row.hidden == 'N' or (dataset_row.hidden == 'Y' and dataset_row.user_id is not None):
                metadata = db.DBSession.query(Metadata).filter(Metadata.dataset_id == dataset_row.id).all()
                dataset_dict['metadata'] = metadata
            else:
                dataset_dict['metadata'] = []

            datasets.append(namedtuple('Dataset', dataset_dict.keys())(**dataset_dict))


    except NoResultFound:
        raise ResourceNotFoundError("Datasets not found.")

    return datasets
165753
def json_qry(dataset, qry_str, params={}):
    
    # if qry_str.startswith("$.bf_itemOf[rdf_type=bf_Print].='print',\n"):
    #     pdb.set_trace()
    if not '$' in qry_str:
        qry_str = ".".join(['$', qry_str.strip()])
    dallor_val = params.get("$", dataset)
    if isinstance(dallor_val, rdflib.URIRef):
        dallor_val = Uri(dallor_val)
    if qry_str.strip() == '$':
        return [dallor_val]
    parsed_qry = parse_json_qry(qry_str)
    qry_parts = parsed_qry['qry_parts']
    post_actions = parsed_qry['params']
    # print(qry_parts)
    rtn_list = UniqueList()
    if params.get('dataset'):
        dataset = params['dataset']
    for or_part in qry_parts:
        if or_part[1] == 0:
            if isinstance(dallor_val, dict):
                result = dallor_val
            else:
                try:
                    result = dataset[dallor_val]
                except KeyError:
                    try:
                        result = dataset[Uri(dallor_val)]
                    except KeyError:
                        try:
                            result = dataset[BlankNode(dallor_val)]
                        except KeyError:
                            continue

            forward = True
            for part in or_part[0][1:]:
                if part == "*":
                    forward = not forward
                else:
                    if forward:
                        result = get_json_qry_item(result, part)
                    else:
                        result = get_reverse_json_qry_item(result,
                                                           part,
                                                           False)
        else:
            result = dataset
            parts = or_part[0].copy()
            parts.reverse()
            forward = False
            for part in parts[1:]:
                if part == "*":
                    forward = not forward
                else:
                    if forward:
                        result = get_json_qry_item(result, part)
                    else:
                        result = get_reverse_json_qry_item(result,
                                                           part,
                                                           False,
                                                           dallor_val)
        rtn_list += result
    for action in post_actions:
        rtn_list = action(rtn_list)
    return rtn_list
467660
def datasets(ctx, client):
    """"""
    from renku.models._jsonld import asjsonld
    from renku.models.datasets import Dataset
    from renku.models.refs import LinkReference

    from ._checks.location_datasets import _dataset_metadata_pre_0_3_4

    for old_path in _dataset_metadata_pre_0_3_4(client):
        with old_path.open('r') as fp:
            dataset = Dataset.from_jsonld(yaml.safe_load(fp))

        name = str(old_path.parent.relative_to(client.path / 'data'))
        new_path = (
            client.renku_datasets_path / dataset.identifier.hex /
            client.METADATA
        )
        new_path.parent.mkdir(parents=True, exist_ok=True)

        dataset = dataset.rename_files(
            lambda key: os.path.
            relpath(str(old_path.parent / key), start=str(new_path.parent))
        )

        with new_path.open('w') as fp:
            yaml.dump(asjsonld(dataset), fp, default_flow_style=False)

        old_path.unlink()

        LinkReference.create(client=client,
                             name='datasets/' + name).set_reference(new_path)
751523
def jsonld(client, datasets):
    """"""
    from renku.models._json import dumps
    from renku.models._jsonld import asjsonld

    data = [
        asjsonld(
            dataset,
            basedir=os.path.relpath(
                '.', start=str(dataset.__reference__.parent)
            )
        ) for dataset in datasets
    ]
    click.echo(dumps(data, indent=2))
767907
def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
                url=None, resume=True, verbose=1):
    
    warn(DEPRECATION_MSG)
    file_id, md5 = OSF_RESOURCES[dataset_name]
    if url is None:
        url = '{}/{}'.format(OSF_PROJECT_URL, file_id)
    return fetch_file(dataset_name, url, data_dir, dataset_prefix=dataset_prefix,
                      filetype='tar', resume=resume, verbose=verbose, md5sum=md5)
10696
def import_json(path, model, batch_len=100, db_alias='default', start_batch=0, end_batch=None, ignore_errors=False, verbosity=2):
    """"""

    # TODO: use a generator to save memory for large json files/databases
    if verbosity > 0:
        print('Reading json records (dictionaries) from {0}.'.format(repr(path)))
    item_list = json.load(open(path, 'r'))
    if verbosity > 0:
        print('Finished reading {0} items from {1}.'.format(len(item_list), repr(path)))
    import_items(item_list, model=model, batch_len=batch_len, db_alias=db_alias, start_batch=start_batch, end_batch=end_batch, ignore_errors=ignore_errors, verbosity=verbosity)
717628
def create(self, dataset_name, query, index_by, display_name):
        
        url = "{0}/{1}".format(self._cached_datasets_url, dataset_name)
        payload = {
            "query": query,
            "index_by": index_by,
            "display_name": display_name
        }
        return self._get_json(HTTPMethods.PUT, url, self._get_master_key(), json=payload)
1219264
def __datasets_desc():
    """"""
    datasets = __get_data_folder_path() + 'datasets.csv'
    df = pd.read_csv(datasets)
    df = df[['Item', 'Title']]
    df.columns = ['dataset_id', 'title']
    # print('a list of the available datasets:')
    return df
520143
def dataset_splits(self):
    """"""
    return [{
        "split": problem.DatasetSplit.TRAIN,
        "shards": self.num_train_shards,
    }, {
        "split": problem.DatasetSplit.EVAL,
        "shards": self.num_eval_shards,
    }, {
        "split": problem.DatasetSplit.TEST,
        "shards": self.num_test_shards,
    }]
1362709
def datasetItems(self):
        
        out = {}
        for item in self.items():
            if isinstance(item, XChartDatasetItem):
                out[item.dataset()] = item
        return out
1183964
def load_json(cls, data, default_rule=None, raise_error=False):
        """"""

        rules = {k: _parser.parse_rule(v, raise_error)
                 for k, v in json.loads(data).items()}

        return cls(rules, default_rule)
103838
def list_datasets():
    """"""
    lines = []
    for name, resource in itertools.chain(LOCAL_DATASETS.items(), REMOTE_DATASETS.items()):

        if isinstance(resource, LocalFileMetadata):
            location = "local: {}".format(resource.filename)
        elif isinstance(resource, RemoteFileMetadata):
            location = "remote: {}".format(resource.url)
        else:
            location = "unknown"
        lines.append("{}\n{}\n{}\n{}".format(name, "=" * len(name), resource.description, location))

    return "\n\n{}\n\n".format(10 * "-").join(lines)
406066
def properties(dataset_uri, item_identifier):
    """"""
    dataset = dtoolcore.DataSet.from_uri(dataset_uri)
    try:
        props = dataset.item_properties(item_identifier)
    except KeyError:
        click.secho(
            "No such item in dataset: {}".format(item_identifier),
            fg="red",
            err=True
        )
        sys.exit(20)

    json_lines = [
        '{',
        '  "relpath": "{}",'.format(props["relpath"]),
        '  "size_in_bytes": {},'.format(props["size_in_bytes"]),
        '  "utc_timestamp": {},'.format(props["utc_timestamp"]),
        '  "hash": "{}"'.format(props["hash"]),
        '}',
    ]
    formatted_json = "\n".join(json_lines)
    colorful_json = pygments.highlight(
        formatted_json,
        pygments.lexers.JsonLexer(),
        pygments.formatters.TerminalFormatter())
    click.secho(colorful_json, nl=False)
807210
def _make_dataset(cls, coords):
        
        class Slice(cls._SliceType):
            extra_coords = coords

        Slice.__name__ = '%s.slice(%s)' % (
            cls.__name__,
            ', '.join('%s=%r' % item for item in coords.items()),
        )
        return Slice