Setup data while splitting into a training, validation, and test set.

        Args:
            X: text data,
            y: data labels,
            tokenizer: A Tokenizer instance
            proc_data_dir: Directory for the split and processed data
168703
def setup_data_split(X, y, tokenizer, proc_data_dir, **kwargs):
    
    X_train, X_val, X_test, y_train, y_val, y_test = split_data(X, y)

    # only build vocabulary on training data
    tokenizer.build_vocab(X_train)

    process_save(X_train, y_train, tokenizer, path.join(
        proc_data_dir, 'train.bin'), train=True, **kwargs)
    process_save(X_val, y_val, tokenizer, path.join(
        proc_data_dir, 'val.bin'), **kwargs)
    process_save(X_test, y_test, tokenizer, path.join(
        proc_data_dir, 'test.bin'), **kwargs)
660537
def set_up_dirs(proc_name, output_dir=None, work_dir=None, log_dir=None):
    
    output_dir = safe_mkdir(adjust_path(output_dir or join(os.getcwd(), proc_name)), 'output_dir')
    debug('Saving results into ' + output_dir)

    work_dir = safe_mkdir(work_dir or join(output_dir, 'work'), 'working directory')
    info('Using work directory ' + work_dir)

    log_fpath = set_up_log(log_dir or safe_mkdir(join(work_dir, 'log')), proc_name + '.log')

    return output_dir, work_dir, log_fpath
448082
def dataproc(cls, graph, data):
        
        for thing in data:
            graph.add_trip(*thing)
        raise NotImplementedError('You need to implement this yourlself!')
494352
def train(self, **kwargs):
        
        log.info("Starting to train...")
        if not self.setup_run:
            self.setup()
        self.trained_tasks = []
        for task in self.tasks:
            data = self.reformatted_input[task.data_format]['data']
            target = self.reformatted_input[task.data_format]['target']
            if data is None:
                raise Exception("Data cannot be none.  Check the config file to make sure the right input is being read.")
            kwargs['data']=data
            kwargs['target']=target
            trained_task = self.execute_train_task_with_dependencies(task, **kwargs)
            self.trained_tasks.append(trained_task)
            #If the trained task alters the data in any way, pass it down the chain to the next task
            if hasattr(trained_task, 'data'):
                self.reformatted_input[task.data_format]['data'] = trained_task.data
        log.info("Finished training.")
64418
def load_mnist_data(args):
    '''
    
    '''
    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train]
    x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test]
    y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train]
    y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test]

    LOG.debug('x_train shape: %s', (x_train.shape,))
    LOG.debug('x_test shape: %s', (x_test.shape,))

    return x_train, y_train, x_test, y_test
478437
def set_data_values(self, label, x, y, z):
        
        # TODO: avoid re-allocating an array every time
        self.layers[label]['data'] = np.array([x, y, z]).transpose()
        self._update()
1060561
def init_train_args(self, parser):
        ''' '''
        parser.add_argument('-f', '--configuration', dest='config', default=DEFAULT_USER_CONFIG_PATH,
                            help='the path to the configuration file to use -- ./config.yaml by default')
        parser.add_argument('-c', '--corpus-name', help='the name of the corpus or saved dataset to train on')
543036
def splits_and_paths(self, data_dir):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    def append_epoch(paths):
      return [
          "{}.{}".format(path, self.current_epoch)
          for path in paths
      ]

    # We set shuffled=True as we don't want to shuffle on disk later.
    return [
        (split["split"], append_epoch(filepath_fns[split["split"]](
            data_dir, split["shards"], shuffled=True
        )))
        for split in self.dataset_splits
    ]
1172792
public static void splitData(final String originalTrainingDataFile) {
        List<Tuple> trainingData = NaiveBayesClassifier.readTrainingData(originalTrainingDataFile, "\\s");
        List<Tuple> wrongData = new ArrayList<>();

        int lastTrainingDataSize;
        int iterCount = 0;
        do {
            System.out.println("Iteration:\t" + (++iterCount));
            lastTrainingDataSize = trainingData.size();

            NaiveBayesClassifier nbc = new NaiveBayesClassifier();
            nbc.train(trainingData);

            Iterator<Tuple> trainingDataIter = trainingData.iterator();
            while (trainingDataIter.hasNext()) {
                Tuple t = trainingDataIter.next();
                String actual = nbc.predictLabel(t);
                if (!t.label.equals(actual) && !t.label.equals("1")) { // preserve 1 since too few.
                    wrongData.add(t);
                    trainingDataIter.remove();
                }
            }

            Iterator<Tuple> wrongDataIter = wrongData.iterator();
            while (wrongDataIter.hasNext()) {
                Tuple t = wrongDataIter.next();
                String actual = nbc.predictLabel(t);
                if (t.label.equals(actual)) {
                    trainingData.add(t);
                    wrongDataIter.remove();
                }
            }
        } while (trainingData.size() != lastTrainingDataSize);

        writeToFile(trainingData, originalTrainingDataFile + ".aligned");
        writeToFile(wrongData, originalTrainingDataFile + ".wrong");
    }
653386
def fit(self, X, y):
        
        self._word_vocab.add_documents(X)
        self._label_vocab.add_documents(y)
        if self._use_char:
            for doc in X:
                self._char_vocab.add_documents(doc)

        self._word_vocab.build()
        self._char_vocab.build()
        self._label_vocab.build()

        return self
699776
def splits(cls, fields, root=".data", train="train.txt",
               test="test.txt", validation_frac=0.1, **kwargs):
        

        train, test = super(CoNLL2000Chunking, cls).splits(
            fields=fields, root=root, train=train,
            test=test, separator=' ', **kwargs)

        # HACK: Saving the sort key function as the split() call removes it
        sort_key = train.sort_key

        # Now split the train set
        # Force a random seed to make the split deterministic
        random.seed(0)
        train, val = train.split(1 - validation_frac, random_state=random.getstate())
        # Reset the seed
        random.seed()

        # HACK: Set the sort key
        train.sort_key = sort_key
        val.sort_key = sort_key

        return train, val, test
331461
def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
    
    generator = self.generate_samples(data_dir, tmp_dir, dataset_split)
    encoder = self.get_or_create_vocab(data_dir, tmp_dir)
    label_encoder = self.get_labels_encoder(data_dir)
    for sample in generator:
      inputs = encoder.encode(sample["inputs"])
      inputs.append(text_encoder.EOS_ID)
      context = encoder.encode(sample["context"])
      context.append(text_encoder.EOS_ID)
      targets = label_encoder.encode(sample["targets"])
      sample["targets"] = targets
      yield {"inputs": inputs, "context": context, "targets": targets}
317884
def train(X_train, y_train, **kwargs):
    '''
    
    '''
    crf = Trainer()
    crf.set_params({
        'c1': 1.0,
        'c2': 0.001,
        'max_iterations': 200,
        'feature.possible_transitions': True,
    })

    for xseq, yseq in zip(X_train, y_train):
        crf.append(xseq, yseq)
    crf.train(TOKENIZATION_MODEL_PATH)
    return crf
117035
def setup_combine(final_file, data):
    
    if "align_split" not in data:
        return final_file, data
    align_dir = os.path.dirname(final_file)
    base, ext = os.path.splitext(os.path.basename(final_file))
    start, end = [int(x) for x in data["align_split"].split("-")]
    out_file = os.path.join(utils.safe_makedir(os.path.join(align_dir, "split")),
                            "%s-%s_%s%s" % (base, start, end, ext))
    data["combine"] = {"work_bam": {"out": final_file, "extras": []}}
    return out_file, data
444253
def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,
               pad_first=True, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        collate_fn = partial(pad_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)
        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs)
        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)
        dataloaders = [train_dl]
        for ds in datasets[1:]:
            lengths = [len(t) for t in ds.x.items]
            sampler = SortSampler(ds.x, key=lengths.__getitem__)
            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))
        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)
311907
def generate_samples(label, num_splits, sampler):
    
    def listify(x):
        return x if isinstance(x, list) else [x]
    label_splits = listify(label.split(num_splits, axis=0))
    prob_samples = []
    prob_targets = []
    samples = []
    for label_split in label_splits:
        label_split_2d = label_split.reshape((-1,1))
        sampled_value = sampler.draw(label_split_2d)
        sampled_classes, exp_cnt_true, exp_cnt_sampled = sampled_value
        samples.append(sampled_classes.astype(np.float32))
        prob_targets.append(exp_cnt_true.astype(np.float32).reshape((-1,1)))
        prob_samples.append(exp_cnt_sampled.astype(np.float32))
    return samples, prob_samples, prob_targets
480479
def init_train_args(self, parser):
        ''' '''
        parser.add_argument('-f', '--configuration', dest='config', default=DEFAULT_USER_CONFIG_PATH,
                            help='the path to the configuration file to use -- ./config.yaml by default')
        parser.add_argument('-c', '--dataset-name', help='the name of the saved dataset to train on')
        parser.add_argument('-s', '--saved-model-name', help='the name of the saved model to continue training')
        parser.add_argument('-n', '--new-model-name', help='the name to give the newly trained model')
        parser.add_argument('-g', '--gpus', dest='num_gpus', type=int, default=1, help='the number of gpus to use during training')