Downloads and prepairs the dataset to be parsed by the data_generator.
768944
def _download_and_parse_dataset(tmp_dir, train):
  """"""
  file_path = generator_utils.maybe_download(tmp_dir, _SNLI_ZIP, _SNLI_URL)
  zip_ref = zipfile.ZipFile(file_path, 'r')
  zip_ref.extractall(tmp_dir)
  zip_ref.close()

  file_name = 'train' if train else 'dev'
  dataset_file_path = os.path.join(tmp_dir, _SNLI_DATA_PATH % file_name)
  _parse_dataset(dataset_file_path, tmp_dir, train)
355120
def run(dataset):
    """"""
    config = _get_config()

    if dataset:
        _print("getting dataset from brains...")
        cprint("done", 'green')

        # check dataset cache for dataset
        # if not exists
        # r = requests.get('https://api.github.com/events', stream=True)
        # with open(filename, 'wb') as fd:
        #     for chunk in r.iter_content(chunk_size):
        #         fd.write(chunk)

    cprint('Running "%s"' % config["run"], 'green', attrs=("underline",))
    call(config["run"].split())
75074
def create_dataset(self, in_stream):
        
        assert XLS_IMPORT
        xls_book = xlrd.open_workbook(file_contents=in_stream)
        dataset = tablib.Dataset()
        sheet = xls_book.sheets()[0]

        dataset.headers = sheet.row_values(0)
        for i in range(1, sheet.nrows):
            dataset.append(sheet.row_values(i))
        return dataset
213983
def load_dataset(data_name):
    """"""
    if data_name == 'MR' or data_name == 'Subj':
        train_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, [])
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths
    else:
        train_dataset, test_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, test_dataset)
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        test_dataset, test_data_lengths = _preprocess_dataset(test_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths, test_dataset, \
               test_data_lengths
562514
@Override
    public void preProcess(DataSet toPreProcess) {
        for (DataSetPreProcessor preProcessor : preProcessors) {
            preProcessor.preProcess(toPreProcess);
        }
    }
176764
def _get_vqa_v2_image_raw_dataset(directory, image_root_url, image_urls):
  """"""
  for url in image_urls:
    filename = os.path.basename(url)
    download_url = os.path.join(image_root_url, url)
    path = generator_utils.maybe_download(directory, filename, download_url)
    unzip_dir = os.path.join(directory, filename.strip(".zip"))
    if not tf.gfile.Exists(unzip_dir):
      zipfile.ZipFile(path, "r").extractall(directory)
79170
def load_dataloader(self):
        

        input_transform = transforms.Compose([transforms.ToTensor(), \
                                             transforms.Normalize((0.7136, 0.4906, 0.3283), \
                                                                  (0.1138, 0.1078, 0.0917))])
        training_dataset = LipsDataset(self.image_path,
                                       self.align_path,
                                       mode='train',
                                       transform=input_transform,
                                       seq_len=self.seq_len)

        self.train_dataloader = mx.gluon.data.DataLoader(training_dataset,
                                                         batch_size=self.batch_size,
                                                         shuffle=True,
                                                         num_workers=self.num_workers)

        valid_dataset = LipsDataset(self.image_path,
                                    self.align_path,
                                    mode='valid',
                                    transform=input_transform,
                                    seq_len=self.seq_len)

        self.valid_dataloader = mx.gluon.data.DataLoader(valid_dataset,
                                                         batch_size=self.batch_size,
                                                         shuffle=True,
                                                         num_workers=self.num_workers)
219776
def datasetsGenerator(self, request):
        
        return self._topLevelObjectGenerator(
            request, self.getDataRepository().getNumDatasets(),
            self.getDataRepository().getDatasetByIndex)
872366
def prepare_data_loader(args, dataset, vocab, test=False):
    
    # Preprocess
    dataset = dataset.transform(lambda s1, s2, label: (vocab(s1), vocab(s2), label),
                                lazy=False)

    # Batching
    batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(), btf.Stack(dtype='int32'))
    data_lengths = [max(len(d[0]), len(d[1])) for d in dataset]
    batch_sampler = nlp.data.FixedBucketSampler(lengths=data_lengths,
                                                batch_size=args.batch_size,
                                                shuffle=(not test))
    data_loader = gluon.data.DataLoader(dataset=dataset,
                                        batch_sampler=batch_sampler,
                                        batchify_fn=batchify_fn)
    return data_loader
30230
def download_data(job, master_ip, inputs, known_snps, bam, hdfs_snps, hdfs_bam):
    

    log.info("Downloading known sites file %s to %s.", known_snps, hdfs_snps)
    call_conductor(job, master_ip, known_snps, hdfs_snps, memory=inputs.memory)

    log.info("Downloading input BAM %s to %s.", bam, hdfs_bam)
    call_conductor(job, master_ip, bam, hdfs_bam, memory=inputs.memory)
1012942
def _cache_dataset(dataset, prefix):
    
    if not os.path.exists(_constants.CACHE_PATH):
        os.makedirs(_constants.CACHE_PATH)
    src_data = np.concatenate([e[0] for e in dataset])
    tgt_data = np.concatenate([e[1] for e in dataset])
    src_cumlen = np.cumsum([0]+[len(e[0]) for e in dataset])
    tgt_cumlen = np.cumsum([0]+[len(e[1]) for e in dataset])
    np.savez(os.path.join(_constants.CACHE_PATH, prefix + '.npz'),
             src_data=src_data, tgt_data=tgt_data,
             src_cumlen=src_cumlen, tgt_cumlen=tgt_cumlen)
385068
def run_parse(self):
        """"""
        # Data set already has source file names from load_inputs
        parsedset = {}
        parsedset['data_set'] = []
        for log in self.input_files:
            parsemodule = self.parse_modules[self.args.parser]
            try:
                if self.args.tzone:
                    parsemodule.tzone = self.args.tzone
            except NameError: pass
            parsedset['data_set'].append(parsemodule.parse_file(log))
        self.data_set = parsedset
        del(parsedset)
975077
def _openResources(self):
        
        logger.info("Opening: {}".format(self._fileName))
        self._ncGroup = Dataset(self._fileName)
156536
def prepare(self):
        """"""
        if not os.path.exists(self.path):
            if self.retrieve:
                print("Downloading SQLite3 database...")
                download_from_url(self.retrieve, self.path, progress=True)
            else: raise Exception("The file '" + self.path + "' does not exist.")
        self.check_format()
        if self.known_md5: assert self.known_md5 == self.md5
        self.prepared = True
946130
def _prepare_lambada_data(tmp_dir, data_dir, vocab_size, vocab_filename):
  

  if not tf.gfile.Exists(data_dir):
    tf.gfile.MakeDirs(data_dir)

  file_path = generator_utils.maybe_download(tmp_dir, _TAR, _URL)
  tar_all = tarfile.open(file_path)
  tar_all.extractall(tmp_dir)
  tar_all.close()
  tar_train = tarfile.open(os.path.join(tmp_dir, "train-novels.tar"))
  tar_train.extractall(tmp_dir)
  tar_train.close()

  vocab_path = os.path.join(data_dir, vocab_filename)
  if not tf.gfile.Exists(vocab_path):
    with tf.gfile.GFile(os.path.join(tmp_dir, _VOCAB), "r") as infile:
      reader = csv.reader(infile, delimiter="\t")
      words = [row[0] for row in reader]
      words = [_UNK] + words[:vocab_size]
    with tf.gfile.GFile(vocab_path, "w") as outfile:
      outfile.write("\n".join(words))
529886
def preprocess_data():
    

    print("Importing dataset from disk...")
    path = 'enron1/ham/'
    ham1 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
            for f in os.listdir(path) if os.path.isfile(path + f)]
    path = 'enron1/spam/'
    spam1 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
             for f in os.listdir(path) if os.path.isfile(path + f)]
    path = 'enron2/ham/'
    ham2 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
            for f in os.listdir(path) if os.path.isfile(path + f)]
    path = 'enron2/spam/'
    spam2 = [open(path + f, 'r', errors='replace').read().strip(r"\n")
             for f in os.listdir(path) if os.path.isfile(path + f)]

    # Merge and create labels
    emails = ham1 + spam1 + ham2 + spam2
    y = np.array([-1] * len(ham1) + [1] * len(spam1) +
                 [-1] * len(ham2) + [1] * len(spam2))

    # Words count, keep only frequent words
    count_vect = CountVectorizer(decode_error='replace', stop_words='english',
                                 min_df=0.001)
    X = count_vect.fit_transform(emails)

    print('Vocabulary size: %d' % X.shape[1])

    # Shuffle
    perm = np.random.permutation(X.shape[0])
    X, y = X[perm, :], y[perm]

    # Split train and test
    split = 500
    X_train, X_test = X[-split:, :], X[:-split, :]
    y_train, y_test = y[-split:], y[:-split]

    print("Labels in trainset are {:.2f} spam : {:.2f} ham".format(
        np.mean(y_train == 1), np.mean(y_train == -1)))

    return X_train, y_train, X_test, y_test
30448
def data_loader(self, sentences, shuffle=False):
        """"""
        dataset = BertEmbeddingDataset(sentences, self.transform)
        return DataLoader(dataset=dataset, batch_size=self.batch_size, shuffle=shuffle)