Loads and preprocessed data for the MR dataset.
    Returns input vectors, labels, vocabulary, and inverse vocabulary.
512059
def load_data_with_word2vec(word2vec_list):
    
    # Load and preprocess data
    sentences, labels = load_data_and_labels()
    sentences_padded = pad_sentences(sentences)
    # vocabulary, vocabulary_inv = build_vocab(sentences_padded)
    return build_input_data_with_word2vec(sentences_padded, labels, word2vec_list)
58848
def cv(row, col_name, arg, current_data_model, df, con):
    
    vocabulary = con.vocab.vocabularies
    cell_value = str(row[col_name])
    if not cell_value:
        return None
    elif cell_value == "None":
        return None
    cell_values = cell_value.split(":")
    cell_values = [c.strip() for c in cell_values]
    # get possible values for controlled vocabulary
    # exclude weird unicode
    possible_values = []
    for val in vocabulary[col_name]:
        try:
            possible_values.append(str(val).lower())
        except UnicodeEncodeError as ex:
            print(val, ex)
    for value in cell_values:
        if str(value).lower() == "nan":
            continue
        elif str(value).lower() in possible_values:
            continue
        elif value.lower() == "none":
            continue
        else:
            try:
                if str(float(value)) in possible_values:
                    continue
            except:
                pass
            return '"{}" is not in controlled vocabulary for {}'.format(value, arg)
    return None
906098
def load_text_data(dataset, directory, centre=True):
    """"""

    points, point_names = parse_text(os.path.join(directory, dataset + '.txt'))[0:2]
    # Remove markers where there is a NaN
    present_index = [i for i in range(points[0].shape[1]) if not (np.any(np.isnan(points[0][:, i])) or np.any(np.isnan(points[0][:, i])) or np.any(np.isnan(points[0][:, i])))]

    point_names = point_names[present_index]
    for i in range(3):
        points[i] = points[i][:, present_index]
        if centre:
            points[i] = (points[i].T - points[i].mean(axis=1)).T 

    # Concatanate the X, Y and Z markers together
    Y = np.concatenate((points[0], points[1], points[2]), axis=1)
    Y = Y/400.
    connect = read_connections(os.path.join(directory, 'connections.txt'), point_names)
    return Y, connect
247653
def load_data_batch(self, data_batch):
        """"""
        _load_data(data_batch, self.data_arrays)
        _load_label(data_batch, self.label_arrays)
782008
def _load(pathtovector,
              wordlist,
              num_to_load=None,
              truncate_embeddings=None,
              sep=" "):
        """"""
        vectors = []
        addedwords = set()
        words = []

        try:
            wordlist = set(wordlist)
        except ValueError:
            wordlist = set()

        logger.info("Loading {0}".format(pathtovector))

        firstline = open(pathtovector).readline().strip()
        try:
            num, size = firstline.split(sep)
            num, size = int(num), int(size)
            logger.info("Vector space: {} by {}".format(num, size))
            header = True
        except ValueError:
            size = len(firstline.split(sep)) - 1
            logger.info("Vector space: {} dim, # items unknown".format(size))
            word, rest = firstline.split(sep, 1)
            # If the first line is correctly parseable, set header to False.
            header = False

        if truncate_embeddings is None or truncate_embeddings == 0:
            truncate_embeddings = size

        for idx, line in enumerate(open(pathtovector, encoding='utf-8')):

            if header and idx == 0:
                continue

            word, rest = line.rstrip(" \n").split(sep, 1)

            if wordlist and word not in wordlist:
                continue

            if word in addedwords:
                raise ValueError("Duplicate: {} on line {} was in the "
                                 "vector space twice".format(word, idx))

            if len(rest.split(sep)) != size:
                raise ValueError("Incorrect input at index {}, size "
                                 "is {}, expected "
                                 "{}".format(idx+1,
                                             len(rest.split(sep)), size))

            words.append(word)
            addedwords.add(word)
            vectors.append(np.fromstring(rest, sep=sep)[:truncate_embeddings])

            if num_to_load is not None and len(addedwords) >= num_to_load:
                break

        vectors = np.array(vectors).astype(np.float32)

        logger.info("Loading finished")
        if wordlist:
            diff = wordlist - addedwords
            if diff:
                logger.info("Not all items from your wordlist were in your "
                            "vector space: {}.".format(diff))

        return vectors, words
810203
def get_input_tensors(batch_size, tf_records, num_repeats=1,
                      shuffle_records=True, shuffle_examples=True,
                      shuffle_buffer_size=None,
                      filter_amount=0.05, random_rotation=True):
    
    print("Reading tf_records from {} inputs".format(len(tf_records)))
    dataset = read_tf_records(
        batch_size,
        tf_records,
        num_repeats=num_repeats,
        shuffle_records=shuffle_records,
        shuffle_examples=shuffle_examples,
        shuffle_buffer_size=shuffle_buffer_size,
        filter_amount=filter_amount,
        interleave=True)
    dataset = dataset.filter(lambda t: tf.equal(tf.shape(t)[0], batch_size))
    dataset = dataset.map(
        functools.partial(batch_parse_tf_example, batch_size))
    if random_rotation:
        dataset = dataset.map(_random_rotation_pyfunc)

    return dataset.make_one_shot_iterator().get_next()
354045
def data():
    
    from keras.datasets import mnist
    from keras.utils import np_utils
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train.reshape(60000, 784)
    x_test = x_test.reshape(10000, 784)
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    nb_classes = 10
    y_train = np_utils.to_categorical(y_train, nb_classes)
    y_test = np_utils.to_categorical(y_test, nb_classes)
    return x_train, y_train, x_test, y_test
336739
def load(self, model):
        """"""
        self.perceptron.weights, self.tagdict, self.classes, self.clusters = load_model(model)
        self.perceptron.classes = self.classes
390017
def input_dataset_from_dataframe(df, delays=(1, 2, 3), inputs=(1, 2, -1), outputs=None, normalize=True, verbosity=1):
    
    return dataset_from_dataframe(df=df, delays=delays, inputs=inputs, outputs=outputs,
                                  normalize=normalize, verbosity=verbosity)
1320050
def load_from_data(self, data, with_undefined=False):
        
        if with_undefined:
            defined_values, undefined_values = data
        else:
            defined_values = data
            undefined_values = None
        self._index = defaultdict(list, defined_values)
        self._reverse_index = defaultdict(list)
        for key, values in self._index.items():
            for value in values:
                self._reverse_index[value].append(key)
        if undefined_values:
            self._undefined_keys = {key: True for key in undefined_values}
        else:
            self._undefined_keys = {}
999902
def _load(self, dataset='train'):
        
        data, labels = None, None
        if dataset is 'train':
            files = [os.path.join(self.cifar10_dir, 'data_batch_%d' % i) for i in range(1, 6)]
        else:
            files = [os.path.join(self.cifar10_dir, 'test_batch')]

        for file in files:
            if not os.path.exists(file):
                raise FileNotFoundError('Failed to find file: ' + file)

        # Load the data from the batch files
        for file in files:
            with open(file, 'rb') as f:
                cifar10 = pickle.load(f, encoding='latin1')

            if labels is None:
                labels = np.array(cifar10['labels'])
            else:
                labels = np.concatenate((labels, cifar10['labels']), axis=0)

            if data is None:
                data = cifar10['data']
            else:
                data = np.concatenate((data, cifar10['data']), axis=0)

        # Adapt the format of the data to our convnet
        data = np.array(data, dtype=float) / 255.0
        data = data.reshape([-1, self.num_channels, self.img_size, self.img_size])
        data = data.transpose([0, 2, 3, 1])

        # One-hot encode labels (see https://stackoverflow.com/a/42874726)
        labels = np.eye(self.num_classes)[np.array(labels).reshape(-1)]

        if dataset is 'train':
            self._train_data, self._train_labels = data, labels
        else:
            self._test_data, self._test_labels = data, labels
945712
def loads(cls, data):
        ''''''
        rep = cbor.loads(data)
        if not isinstance(rep, Sequence):
            raise SerializationError('expected a CBOR list')
        if len(rep) != 2:
            raise SerializationError('expected a CBOR list of 2 items')
        metadata = rep[0]
        if 'v' not in metadata:
            raise SerializationError('no version in CBOR metadata')
        if metadata['v'] != 'fc01':
            raise SerializationError('invalid CBOR version {!r} '
                                     '(expected "fc01")'
                                     .format(metadata['v']))
        read_only = metadata.get('ro', False)
        contents = rep[1]
        return cls.from_dict(contents, read_only=read_only)
885493
def loadRes(self, resFile):
        
        res = COCO()
        res.dataset['images'] = [img for img in self.dataset['images']]

        print('Loading and preparing results...')
        tic = time.time()
        if type(resFile) == str: #or type(resFile) == unicode:
            anns = json.load(open(resFile))
        elif type(resFile) == np.ndarray:
            anns = self.loadNumpyAnnotations(resFile)
        else:
            anns = resFile
        assert type(anns) == list, 'results in not an array of objects'
        annsImgIds = [ann['image_id'] for ann in anns]
        assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \
               'Results do not correspond to current coco set'
        if 'caption' in anns[0]:
            imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns])
            res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds]
            for id, ann in enumerate(anns):
                ann['id'] = id+1
        elif 'bbox' in anns[0] and not anns[0]['bbox'] == []:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                ann['area'] = bb[2]*bb[3]
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'segmentation' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                # now only support compressed RLE format as segmentation results
                ann['area'] = maskUtils.area(ann['segmentation'])
                if not 'bbox' in ann:
                    ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'keypoints' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                s = ann['keypoints']
                x = s[0::3]
                y = s[1::3]
                x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y)
                ann['area'] = (x1-x0)*(y1-y0)
                ann['id'] = id + 1
                ann['bbox'] = [x0,y0,x1-x0,y1-y0]
        print('DONE (t={:0.2f}s)'.format(time.time()- tic))

        res.dataset['annotations'] = anns
        res.createIndex()
        return res
887184
def load_character_images(self, verbose=True):
        
        for dataset in self.character_sets:
            assert self.get_dataset(dataset) is True, "Datasets aren't properly downloaded, " \
                                                      "rerun to try again or download datasets manually."

        for dataset in self.character_sets:
            for image, label in self.load_dataset(dataset, verbose=verbose):
                yield image, label
587146
def get_vocabulary(preprocess_output_dir, name):
  
  vocab_file = os.path.join(preprocess_output_dir, CATEGORICAL_ANALYSIS % name)
  if not file_io.file_exists(vocab_file):
    raise ValueError('File %s not found in %s' %
                     (CATEGORICAL_ANALYSIS % name, preprocess_output_dir))

  labels = python_portable_string(
      file_io.read_file_to_string(vocab_file)).split('\n')
  label_values = [x for x in labels if x]  # remove empty lines

  return label_values
1006959
def _get_data(self, id_list, format='MLDataset'):
        """"""

        format = format.lower()

        features = list()  # returning a dict would be better if AutoMKL() can handle it
        for modality, data in self._modalities.items():
            if format in ('ndarray', 'data_matrix'):
                # turning dict of arrays into a data matrix
                # this is arguably worse, as labels are difficult to pass
                subset = np.array(itemgetter(*id_list)(data))
            elif format in ('mldataset', 'pyradigm'):
                # getting container with fake data
                subset = self._dataset.get_subset(id_list)
                # injecting actual features
                subset.data = { id_: data[id_] for id_ in id_list }
            else:
                raise ValueError('Invalid output format - choose only one of '
                                 'MLDataset or data_matrix')

            features.append(subset)

        return features
172715
def _load_data(self,
                   equities,
                   futures,
                   exchanges,
                   root_symbols,
                   equity_supplementary_mappings):
        
        # Set named identifier columns as indices, if provided.
        _normalize_index_columns_in_place(
            equities=equities,
            equity_supplementary_mappings=equity_supplementary_mappings,
            futures=futures,
            exchanges=exchanges,
            root_symbols=root_symbols,
        )

        futures_output = self._normalize_futures(futures)

        equity_supplementary_mappings_output = (
            self._normalize_equity_supplementary_mappings(
                equity_supplementary_mappings,
            )
        )

        exchanges_output = _generate_output_dataframe(
            data_subset=exchanges,
            defaults=_exchanges_defaults,
        )

        equities_output, equities_mappings = self._normalize_equities(
            equities,
            exchanges_output,
        )

        root_symbols_output = _generate_output_dataframe(
            data_subset=root_symbols,
            defaults=_root_symbols_defaults,
        )

        return AssetData(
            equities=equities_output,
            equities_mappings=equities_mappings,
            futures=futures_output,
            exchanges=exchanges_output,
            root_symbols=root_symbols_output,
            equity_supplementary_mappings=equity_supplementary_mappings_output,
        )