Download and parse MNIST dataset.
178823
def dataset(directory, images_file, labels_file):
  """"""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))
552436
def main(unused_argv):
  """"""
  tf.logging.set_verbosity(tf.logging.INFO)

  make_dir(FLAGS.raw_dir)
  make_dir(FLAGS.data_dir)

  # Get paths of download/extracted training and evaluation files.
  tf.logging.info("Step 1/4: Downloading data from source")
  train_files = get_raw_files(FLAGS.raw_dir, _TRAIN_DATA_SOURCES)
  eval_files = get_raw_files(FLAGS.raw_dir, _EVAL_DATA_SOURCES)

  # Create subtokenizer based on the training files.
  tf.logging.info("Step 2/4: Creating subtokenizer and building vocabulary")
  train_files_flat = train_files["inputs"] + train_files["targets"]
  vocab_file = os.path.join(FLAGS.data_dir, VOCAB_FILE)
  subtokenizer = tokenizer.Subtokenizer.init_from_files(
      vocab_file, train_files_flat, _TARGET_VOCAB_SIZE, _TARGET_THRESHOLD,
      min_count=None if FLAGS.search else _TRAIN_DATA_MIN_COUNT)

  tf.logging.info("Step 3/4: Compiling training and evaluation data")
  compiled_train_files = compile_files(FLAGS.raw_dir, train_files, _TRAIN_TAG)
  compiled_eval_files = compile_files(FLAGS.raw_dir, eval_files, _EVAL_TAG)

  # Tokenize and save data as Examples in the TFRecord format.
  tf.logging.info("Step 4/4: Preprocessing and saving data")
  train_tfrecord_files = encode_and_save_files(
      subtokenizer, FLAGS.data_dir, compiled_train_files, _TRAIN_TAG,
      _TRAIN_SHARDS)
  encode_and_save_files(
      subtokenizer, FLAGS.data_dir, compiled_eval_files, _EVAL_TAG,
      _EVAL_SHARDS)

  for fname in train_tfrecord_files:
    shuffle_records(fname)
354045
def data():
    
    from keras.datasets import mnist
    from keras.utils import np_utils
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train.reshape(60000, 784)
    x_test = x_test.reshape(10000, 784)
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    nb_classes = 10
    y_train = np_utils.to_categorical(y_train, nb_classes)
    y_test = np_utils.to_categorical(y_test, nb_classes)
    return x_train, y_train, x_test, y_test
115847
def load_data_and_labels():
    
    # download dataset
    get_chinese_text()

    # Load data from files
    positive_examples = list(codecs.open("./data/pos.txt", "r", "utf-8").readlines())
    positive_examples = [s.strip() for s in positive_examples]
    positive_examples = [pe for pe in positive_examples if len(pe) < 100]
    negative_examples = list(codecs.open("./data/neg.txt", "r", "utf-8").readlines())
    negative_examples = [s.strip() for s in negative_examples]
    negative_examples = [ne for ne in negative_examples if len(ne) < 100]
    # Split by words
    x_text = positive_examples + negative_examples
    # x_text = [clean_str(sent) for sent in x_text]
    x_text = [list(s) for s in x_text]

    # Generate labels
    positive_labels = [[0, 1] for _ in positive_examples]
    negative_labels = [[1, 0] for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]
317876
def _extract_mnist_images(filename, num_images):
  
  with gzip.open(filename) as bytestream:
    bytestream.read(16)
    buf = bytestream.read(_MNIST_IMAGE_SIZE * _MNIST_IMAGE_SIZE * num_images)
    data = np.frombuffer(buf, dtype=np.uint8)
    data = data.reshape(num_images, _MNIST_IMAGE_SIZE, _MNIST_IMAGE_SIZE, 1)
  return data
686896
def _get_fashion_mnist(directory):
  """"""
  # Fashion mnist files have the same names as MNIST.
  # We must choose a separate name (by adding 'fashion-' prefix) in the tmp_dir.
  for filename in [
      _MNIST_TRAIN_DATA_FILENAME, _MNIST_TRAIN_LABELS_FILENAME,
      _MNIST_TEST_DATA_FILENAME, _MNIST_TEST_LABELS_FILENAME
  ]:
    generator_utils.maybe_download(directory,
                                   _FASHION_MNIST_LOCAL_FILE_PREFIX + filename,
                                   _FASHION_MNIST_URL + filename)
468225
def get_dataset(self, dataset_id, ds_info):
        """"""
        msg = self._get_message(ds_info)
        ds_info = self.get_metadata(msg, ds_info)
        fill = msg['missingValue']
        data = msg.values.astype(np.float32)
        if msg.valid_key('jScansPositively') and msg['jScansPositively'] == 1:
            data = data[::-1]

        if isinstance(data, np.ma.MaskedArray):
            data = data.filled(np.nan)
            data = da.from_array(data, chunks=CHUNK_SIZE)
        else:
            data[data == fill] = np.nan
            data = da.from_array(data, chunks=CHUNK_SIZE)

        return xr.DataArray(data, attrs=ds_info, dims=('y', 'x'))
703047
def load_data_and_labels():
    
    # Load data from files
    pos_path = "./data/rt-polaritydata/rt-polarity.pos"
    neg_path = "./data/rt-polaritydata/rt-polarity.neg"
    if not os.path.exists(pos_path):
        os.system("git clone https://github.com/dennybritz/cnn-text-classification-tf.git")
        os.system('mv cnn-text-classification-tf/data .')
        os.system('rm -rf cnn-text-classification-tf')
    positive_examples = list(open(pos_path).readlines())
    positive_examples = [s.strip() for s in positive_examples]
    negative_examples = list(open(neg_path).readlines())
    negative_examples = [s.strip() for s in negative_examples]
    # Split by words
    x_text = positive_examples + negative_examples
    x_text = [clean_str(sent) for sent in x_text]
    x_text = [s.split(" ") for s in x_text]
    # Generate labels
    positive_labels = [1 for _ in positive_examples]
    negative_labels = [0 for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]
64418
def load_mnist_data(args):
    '''
    
    '''
    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train]
    x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test]
    y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train]
    y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test]

    LOG.debug('x_train shape: %s', (x_train.shape,))
    LOG.debug('x_test shape: %s', (x_test.shape,))

    return x_train, y_train, x_test, y_test
737633
def main():
    """"""
    # Read config file. This has to get updated via git
    project_root = utils.get_project_root()
    infofile = os.path.join(project_root, "raw-datasets/info.yml")
    logging.info("Read '%s'...", infofile)
    with open(infofile, 'r') as ymlfile:
        datasets = yaml.load(ymlfile)
    for dataset in datasets:
        local_path_file = os.path.join(project_root, dataset['online_path'])
        i = 0
        while not is_file_consistent(local_path_file, dataset['md5']) and i < 3:
            if os.path.isfile(local_path_file):
                local_file_size = os.path.getsize(local_path_file)
                logging.info("MD5 codes differ. ")
                logging.info("The file size of the downloaded file is %s.",
                             utils.sizeof_fmt(local_file_size))
            logging.info("Download the file '%s'...", dataset['online_path'])
            urllib.urlretrieve(dataset['url'], local_path_file)
            i += 1
        if i < 10:
            logging.info("Found '%s'.", dataset['online_path'])
620135
def _parse_saved_model(path):
  """"""
  # Based on tensorflow/python/saved_model/loader.py implementation.
  path_to_pb = _get_saved_model_proto_path(path)
  file_content = tf_v1.gfile.Open(path_to_pb, "rb").read()
  saved_model = saved_model_pb2.SavedModel()
  try:
    saved_model.ParseFromString(file_content)
  except message.DecodeError as e:
    raise IOError("Cannot parse file %s: %s." % (path_to_pb, str(e)))
  return saved_model
881708
def mnist(training):
  """"""
  if training:
    data_filename = 'train-images-idx3-ubyte.gz'
    labels_filename = 'train-labels-idx1-ubyte.gz'
    count = 60000
  else:
    data_filename = 't10k-images-idx3-ubyte.gz'
    labels_filename = 't10k-labels-idx1-ubyte.gz'
    count = 10000
  data_filename = maybe_download(MNIST_URL, data_filename)
  labels_filename = maybe_download(MNIST_URL, labels_filename)

  return (mnist_extract_data(data_filename, count),
          mnist_extract_labels(labels_filename, count))
992043
def train_classifier(download=True, parameters=None, ngram_range=(1, 1)):
    """"""
    if download:
        download_wiki()

    path = os.path.join(l.TOPDIR, 'train.json')
    training_set = json.load(open(path))
    path = os.path.join(l.TOPDIR, 'wiki.json')
    wiki_set = json.load(open(path))

    target_names = list(set([i['unit'] for i in training_set + wiki_set]))
    train_data, train_target = [], []
    for example in training_set + wiki_set:
        train_data.append(clean_text(example['text']))
        train_target.append(target_names.index(example['unit']))

    tfidf_model = TfidfVectorizer(sublinear_tf=True,
                                  ngram_range=ngram_range,
                                  stop_words='english')

    matrix = tfidf_model.fit_transform(train_data)

    if parameters is None:
        parameters = {'loss': 'log', 'penalty': 'l2', 'n_iter': 50,
                      'alpha': 0.00001, 'fit_intercept': True}

    clf = SGDClassifier(**parameters).fit(matrix, train_target)
    obj = {'tfidf_model': tfidf_model,
           'clf': clf,
           'target_names': target_names}
    path = os.path.join(l.TOPDIR, 'clf.pickle')
    pickle.dump(obj, open(path, 'w'))
787311
def get_dataset(self, key, info):
        """"""
        logger.debug("Getting raw data")
        res = super(HRITGOESFileHandler, self).get_dataset(key, info)

        self.mda['calibration_parameters'] = self._get_calibration_params()

        res = self.calibrate(res, key.calibration)
        new_attrs = info.copy()
        new_attrs.update(res.attrs)
        res.attrs = new_attrs
        res.attrs['platform_name'] = self.platform_name
        res.attrs['sensor'] = 'goes_imager'
        return res
663476
def download_data(dataset_name=None, prompt=prompt_stdin):
    """"""
        
    dr = data_resources[dataset_name]
    if not authorize_download(dataset_name, prompt=prompt):
        raise Exception("Permission to download data set denied.")
    
    if 'suffices' in dr:
        for url, files, suffices in zip(dr['urls'], dr['files'], dr['suffices']):
            for file, suffix in zip(files, suffices):
                download_url(url=os.path.join(url,file),
                             dir_name = data_path,
                             store_directory=dataset_name,
                             suffix=suffix)
    elif 'dirs' in dr:
        for url, dirs, files in zip(dr['urls'], dr['dirs'], dr['files']):
            for file, dir in zip(files, dirs):
                print(file, dir)
                download_url(
                    url=os.path.join(url,dir,file),
                    dir_name = data_path,
                    store_directory=os.path.join(dataset_name,dir)
                    )
    else:
        for url, files in zip(dr['urls'], dr['files']):
            for file in files:
                download_url(
                    url=os.path.join(url,file),
                    dir_name = data_path,
                    store_directory=dataset_name
                    )
    return True
925001
def get_pretrain_dataset(data, batch_size, num_ctxes, shuffle, use_avg_len,
                         num_buckets, num_parts=1, part_idx=0, prefetch=True):
    """"""
    num_files = len(glob.glob(os.path.expanduser(data)))
    logging.debug('%d files found.', num_files)
    assert num_files >= num_parts, \
        'Number of training files must be greater than the number of partitions'
    split_sampler = nlp.data.SplitSampler(num_files, num_parts=num_parts, part_index=part_idx)
    stream = nlp.data.SimpleDatasetStream(nlp.data.NumpyDataset, data, split_sampler)
    if prefetch:
        stream = nlp.data.PrefetchingStream(stream)

    def get_dataloader(dataset):
        """create data loader based on the dataset chunk"""
        lengths = dataset.get_field('valid_lengths')
        # A batch includes: input_id, masked_id, masked_position, masked_weight,
        #                   next_sentence_label, segment_id, valid_length
        batchify_fn = Tuple(Pad(), Pad(), Pad(), Pad(), Stack(), Pad(), Stack())
        if use_avg_len:
            # sharded data loader
            sampler = nlp.data.FixedBucketSampler(lengths=lengths,
                                                  # batch_size per shard
                                                  batch_size=batch_size,
                                                  num_buckets=num_buckets,
                                                  shuffle=shuffle,
                                                  use_average_length=True,
                                                  num_shards=num_ctxes)
            dataloader = nlp.data.ShardedDataLoader(dataset,
                                                    batch_sampler=sampler,
                                                    batchify_fn=batchify_fn,
                                                    num_workers=num_ctxes)
        else:
            sampler = nlp.data.FixedBucketSampler(lengths,
                                                  batch_size=batch_size * num_ctxes,
                                                  num_buckets=num_buckets,
                                                  ratio=0,
                                                  shuffle=shuffle)
            dataloader = DataLoader(dataset=dataset,
                                    batch_sampler=sampler,
                                    batchify_fn=batchify_fn,
                                    num_workers=1)
        logging.debug('Sampler created for a new dataset:\n%s', sampler.stats())
        return dataloader

    stream = stream.transform(get_dataloader)
    return stream
222073
def get_dataset(self, dataset):
        
        # If the dataset is present, no need to download anything.
        success = True
        dataset_path = self.base_dataset_path + dataset
        if not isdir(dataset_path):

            # Try 5 times to download. The download page is unreliable, so we need a few tries.
            was_error = False
            for iteration in range(5):

                # Guard against trying again if successful
                if iteration == 0 or was_error is True:
                    zip_path = dataset_path + ".zip"

                    # Download zip files if they're not there
                    if not isfile(zip_path):
                        try:
                            with DLProgress(unit='B', unit_scale=True, miniters=1, desc=dataset) as pbar:
                                urlretrieve(self.datasets[dataset]["url"], zip_path, pbar.hook)
                        except Exception as ex:
                            print("Error downloading %s: %s" % (dataset, ex))
                            was_error = True

                    # Unzip the data files
                    if not isdir(dataset_path):
                        try:
                            with zipfile.ZipFile(zip_path) as zip_archive:
                                zip_archive.extractall(path=dataset_path)
                                zip_archive.close()
                        except Exception as ex:
                            print("Error unzipping %s: %s" % (zip_path, ex))
                            # Usually the error is caused by a bad zip file.
                            # Delete it so the program will try to download it again.
                            try:
                                remove(zip_path)
                            except FileNotFoundError:
                                pass
                            was_error = True

            if was_error:
                print("\nThis recognizer is trained by the CASIA handwriting database.")
                print("If the download doesn't work, you can get the files at %s" % self.datasets[dataset]["url"])
                print("If you have download problems, "
                      "wget may be effective at downloading because of download resuming.")
                success = False

        return success