Trains on candidate pairs extracted from the corpus
343564
public boolean trainOnCandidates() {

		List<String> corpus = loadTrainingCorpus(_trainingCorpusFile);

		List<List<Acronym>> trainingExtractedCandidates = new ArrayList<List<Acronym>>();

		for (Integer docID = 0; docID < corpus.size(); ++docID) {
			// Adds to training examples, all the extracted pairs from the
			// current document.
			trainingExtractedCandidates.add(extractCandidatePairs(corpus
					.get(docID)));
		}

		return _abbvHmm.train(trainingExtractedCandidates, null, true);
	}
1180998
def train(args):
    """"""
    vocab, row, col, counts = get_train_data(args)
    model = GloVe(token_to_idx=vocab.token_to_idx, output_dim=args.emsize,
                  dropout=args.dropout, x_max=args.x_max, alpha=args.alpha,
                  weight_initializer=mx.init.Uniform(scale=1 / args.emsize))
    context = get_context(args)
    model.initialize(ctx=context)
    if not args.no_hybridize:
        model.hybridize(static_alloc=not args.no_static_alloc)

    optimizer_kwargs = dict(learning_rate=args.lr, eps=args.adagrad_eps)
    params = list(model.collect_params().values())
    try:
        trainer = mx.gluon.Trainer(params, 'groupadagrad', optimizer_kwargs)
    except ValueError:
        logging.warning('MXNet <= v1.3 does not contain '
                        'GroupAdaGrad support. Falling back to AdaGrad')
        trainer = mx.gluon.Trainer(params, 'adagrad', optimizer_kwargs)

    index_dtype = 'int32'
    if counts.shape[0] >= np.iinfo(np.int32).max:
        index_dtype = 'int64'
        logging.info('Co-occurrence matrix is large. '
                     'Using int64 to represent sample indices.')
    indices = mx.nd.arange(counts.shape[0], dtype=index_dtype)
    for epoch in range(args.epochs):
        # Logging variables
        log_wc = 0
        log_start_time = time.time()
        log_avg_loss = 0

        mx.nd.shuffle(indices, indices)  # inplace shuffle
        bs = args.batch_size
        num_batches = indices.shape[0] // bs
        for i in range(num_batches):
            batch_indices = indices[bs * i:bs * (i + 1)]
            ctx = context[i % len(context)]
            batch_row = row[batch_indices].as_in_context(ctx)
            batch_col = col[batch_indices].as_in_context(ctx)
            batch_counts = counts[batch_indices].as_in_context(ctx)
            with mx.autograd.record():
                loss = model(batch_row, batch_col, batch_counts)
                loss.backward()

            if len(context) == 1 or (i + 1) % len(context) == 0:
                trainer.step(batch_size=1)

            # Logging
            log_wc += loss.shape[0]
            log_avg_loss += loss.mean().as_in_context(context[0])
            if (i + 1) % args.log_interval == 0:
                # Forces waiting for computation by computing loss value
                log_avg_loss = log_avg_loss.asscalar() / args.log_interval
                wps = log_wc / (time.time() - log_start_time)
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'.format(
                                 epoch, i + 1, num_batches, log_avg_loss,
                                 wps / 1000, log_wc / 1000))
                log_dict = dict(
                    global_step=epoch * len(indices) + i * args.batch_size,
                    epoch=epoch, batch=i + 1, loss=log_avg_loss,
                    wps=wps / 1000)
                log(args, log_dict)

                log_start_time = time.time()
                log_avg_loss = 0
                log_wc = 0

            if args.eval_interval and (i + 1) % args.eval_interval == 0:
                with print_time('mx.nd.waitall()'):
                    mx.nd.waitall()
                with print_time('evaluate'):
                    evaluate(args, model, vocab, i + num_batches * epoch)

    # Evaluate
    with print_time('mx.nd.waitall()'):
        mx.nd.waitall()
    with print_time('evaluate'):
        evaluate(args, model, vocab, num_batches * args.epochs,
                 eval_analogy=not args.no_eval_analogy)

    # Save params
    with print_time('save parameters'):
        model.save_parameters(os.path.join(args.logdir, 'glove.params'))
692834
def train(self):
        ''''''

        # Initialize the train subcommand's argparser
        parser = argparse.ArgumentParser(description='Train a dialogue model on a dialogue corpus or a dsrt dataset')
        self.init_train_args(parser)

        # Parse the args we got
        args = parser.parse_args(sys.argv[2:])
        args.config = ConfigurationLoader(args.config).load().model_config

        print(CLI_DIVIDER + '\n')
        Trainer(**vars(args)).run()
755354
def tokenize(qp_pair, tokenizer=None, is_training=False):
    '''
    
    '''
    question_tokens = tokenizer.tokenize(qp_pair['question'])
    passage_tokens = tokenizer.tokenize(qp_pair['passage'])
    if is_training:
        question_tokens = question_tokens[:300]
        passage_tokens = passage_tokens[:300]
    passage_tokens.insert(
        0, {'word': '<BOS>', 'original_text': '<BOS>', 'char_begin': 0, 'char_end': 0})
    passage_tokens.append(
        {'word': '<EOS>', 'original_text': '<EOS>', 'char_begin': 0, 'char_end': 0})
    qp_pair['question_tokens'] = question_tokens
    qp_pair['passage_tokens'] = passage_tokens
1041818
def train(self):
    
    for iDriving, cDriving in enumerate(self.drivingOperandSDRs):
      minicolumnSDR = self.minicolumnSDRs[iDriving]
      self.pairLayerProximalConnections.associate(minicolumnSDR, cDriving)
      for iContext, cContext in enumerate(self.contextOperandSDRs):
        iResult = (iContext + iDriving) % self.numLocations
        cResult = self.resultSDRs[iResult]
        self.pairLayer.compute(minicolumnSDR, basalInput=cContext)
        cPair = self.pairLayer.getWinnerCells()
        self.poolingLayer.associate(cResult, cPair)
251674
def preprocess_example_common(example, mode, hparams):
  """"""
  if "inputs" in example and hparams.max_input_seq_length > 0:
    example["inputs"] = example["inputs"][:hparams.max_input_seq_length]
  if hparams.prepend_mode != "none":
    if mode == tf.estimator.ModeKeys.PREDICT:
      example["partial_targets"] = tf.concat([example["inputs"], [0]], 0)
    else:
      example["targets"] = tf.concat(
          [example["inputs"], [0], example["targets"]], 0)
  if "targets" in example and hparams.max_target_seq_length > 0:
    example["targets"] = example["targets"][:hparams.max_target_seq_length]
  if hparams.split_to_length:
    new_example = {}
    for k, v in six.iteritems(example):
      if k == "targets" or k == "inputs":
        new_example[k] = tf.reshape(v, [-1, hparams.split_to_length, 1, 1])
      else:
        tf.logging.warning("Dropping feature %s" % k)
    return tf.data.Dataset.from_tensor_slices(new_example)
  return example
762718
def _train_lbfgs(self, X_feat_train, X_seq_train, y_train,
                     X_feat_valid, X_seq_valid, y_valid,
                     graph, var, other_var,
                     early_stop_patience=None,
                     n_cores=3):
        
        tic = time.time()
        # take out the parameters for conveience
        n_epochs = self._param["n_epochs"]
        print_every = self._param["print_every"]
        step_size = self._param["step_size"]
        num_steps = n_epochs

        print('Number of epochs:', n_epochs)
        # print("Number of steps per epoch:", num_steps)
        # print("Number of total steps:", num_steps * n_epochs)

        # move into the graph and start the model
        loss_history = []
        train_acc_vec = []
        valid_acc_vec = []
        step_history = []

        with tf.Session(graph=graph, config=tf.ConfigProto(
                use_per_session_threads=True,
                inter_op_parallelism_threads=n_cores,
                intra_op_parallelism_threads=n_cores)) as sess:

            sess.run(other_var["init"])

            best_performance = None
            best_performance_epoch = 0
            for step in range(n_epochs):
                # run the model (sess.run)
                # compute the optimizer, loss and train_prediction in the graph
                # save the last two as l and predictions

                # put thet data into TF form:
                feed_dict = {other_var["tf_X_seq"]: X_seq_train, other_var["tf_y"]: y_train,
                             other_var["tf_X_feat"]: X_feat_train,
                             other_var["tf_step_size"]: step_size}

                # run the optimizer
                # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/opt/python/training/external_optimizer.py#L115
                other_var["optimizer"].minimize(sess, feed_dict=feed_dict)
                l = sess.run(other_var["loss"], feed_dict=feed_dict)
                loss_history.append(l)  # keep storing the full loss history

                # sometimes print the actual training prediction (l)
                if (step % print_every == 0):
                    train_accuracy = self._accuracy_in_session(sess, other_var,
                                                               X_feat_train, X_seq_train, y_train)
                    valid_accuracy = self._accuracy_in_session(sess, other_var,
                                                               X_feat_valid, X_seq_valid, y_valid)
                    # append the prediction accuracies
                    train_acc_vec.append(train_accuracy)
                    valid_acc_vec.append(valid_accuracy)
                    step_history.append(step / num_steps)
                    print('Step %4d: loss %f, train mse: %f, validation mse: %f' %
                          (step, l, train_accuracy, valid_accuracy))
                    # check if this is the best accuracy
                    if best_performance is None or valid_accuracy <= best_performance:
                        best_performance = valid_accuracy
                        best_performance_epoch = step

                if early_stop_patience is not None and step > best_performance_epoch + early_stop_patience:
                    print("Early stopping. best_performance_epoch: %d, best_performance: %f" %
                          (best_performance_epoch, best_performance))
                    break

            # get the test accuracies
            train_accuracy_final = self._accuracy_in_session(sess, other_var,
                                                             X_feat_train, X_seq_train, y_train)
            valid_accuracy_final = self._accuracy_in_session(sess, other_var,
                                                             X_feat_valid, X_seq_valid, y_valid)
            print('Validation accuracy final: %f' % valid_accuracy_final)

            # store the fitted weights
            var_res = self._get_var_res_sess(sess, var)

            # store also the quasi splines fit

            if self._param["n_splines"] is not None:
                self._splines["quasi_X"] = [self._predict_in_session(sess, other_var,
                                                                     X_feat_train[i:(i + 1)],
                                                                     X_seq_train[i:(i + 1)],
                                                                     variable="spline_quasi_X")
                                            for i in range(X_feat_train.shape[0])]
                # transform into the appropriate form
                self._splines["quasi_X"] = np.concatenate([x[0][np.newaxis] for x in self._splines["quasi_X"]])

            accuracy = {
                "loss_history": np.array(loss_history),
                "step_history": np.array(step_history),
                "train_acc_history": np.array(train_acc_vec),
                "val_acc_history": np.array(valid_acc_vec),
                "train_acc_final": train_accuracy_final,
                "val_acc_final": valid_accuracy_final,
                "best_val_acc": best_performance,
                "best_val_acc_epoch": best_performance_epoch,
                "test_acc_final": None,  # test_accuracy_final,
                "y_test": None,  # y_test,
                "y_test_prediction": None,  # test_prediction.eval(),
                "id_vec_test": None  # id_vec_test
            }
            self._accuracy = accuracy

        toc = time.time()
        exec_time = toc - tic
        self._exec_time = exec_time
        print('That took %fs' % exec_time)
        # weights = {"motif_base_weights": motif_base_weights,
        #            "motif_weights": motif_weights,
        #            "motif_bias": motif_bias,
        #            "final_bias": final_bias,
        #            "feature_weights": feature_weights,
        #            "spline_pred": spline_pred
        #            }
        return var_res
832712
def train(epochs, ctx):
    """"""
    if isinstance(ctx, mx.Context):
        ctx = [ctx]
    net.initialize(mx.init.Xavier(magnitude=2), ctx=ctx)

    opt_options = {'learning_rate': opt.lr, 'wd': opt.wd}
    if opt.optimizer == 'sgd':
        opt_options['momentum'] = 0.9
    if opt.optimizer == 'adam':
        opt_options['epsilon'] = 1e-7
    trainer = gluon.Trainer(net.collect_params(), opt.optimizer,
                            opt_options,
                            kvstore=opt.kvstore)
    if opt.lr_beta > 0.0:
        # Jointly train class-specific beta.
        # See "sampling matters in deep embedding learning" paper for details.
        beta.initialize(mx.init.Constant(opt.beta), ctx=ctx)
        trainer_beta = gluon.Trainer([beta], 'sgd',
                                     {'learning_rate': opt.lr_beta, 'momentum': 0.9},
                                     kvstore=opt.kvstore)

    loss = MarginLoss(margin=opt.margin, nu=opt.nu)

    best_val = 0.0
    for epoch in range(epochs):
        tic = time.time()
        prev_loss, cumulative_loss = 0.0, 0.0

        # Learning rate schedule.
        trainer.set_learning_rate(get_lr(opt.lr, epoch, steps, opt.factor))
        logging.info('Epoch %d learning rate=%f', epoch, trainer.learning_rate)
        if opt.lr_beta > 0.0:
            trainer_beta.set_learning_rate(get_lr(opt.lr_beta, epoch, steps, opt.factor))
            logging.info('Epoch %d beta learning rate=%f', epoch, trainer_beta.learning_rate)

        # Inner training loop.
        for i in range(200):
            batch = train_data.next()
            data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0)
            label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0)

            Ls = []
            with ag.record():
                for x, y in zip(data, label):
                    a_indices, anchors, positives, negatives, _ = net(x)

                    if opt.lr_beta > 0.0:
                        L = loss(anchors, positives, negatives, beta, y[a_indices])
                    else:
                        L = loss(anchors, positives, negatives, opt.beta, None)

                    # Store the loss and do backward after we have done forward
                    # on all GPUs for better speed on multiple GPUs.
                    Ls.append(L)
                    cumulative_loss += nd.mean(L).asscalar()

                for L in Ls:
                    L.backward()

            # Update.
            trainer.step(batch.data[0].shape[0])
            if opt.lr_beta > 0.0:
                trainer_beta.step(batch.data[0].shape[0])

            if (i+1) % opt.log_interval == 0:
                logging.info('[Epoch %d, Iter %d] training loss=%f' % (
                    epoch, i+1, cumulative_loss - prev_loss))
                prev_loss = cumulative_loss

        logging.info('[Epoch %d] training loss=%f'%(epoch, cumulative_loss))
        logging.info('[Epoch %d] time cost: %f'%(epoch, time.time()-tic))

        names, val_accs = test(ctx)
        for name, val_acc in zip(names, val_accs):
            logging.info('[Epoch %d] validation: %s=%f'%(epoch, name, val_acc))

        if val_accs[0] > best_val:
            best_val = val_accs[0]
            logging.info('Saving %s.' % opt.save_model_prefix)
            net.save_parameters('%s.params' % opt.save_model_prefix)
    return best_val
161818
public void learn(RNNSearch<double[], double[]> range, double[][] samples, double radius, int Nz, double sigma) {
        TrainSample[] training = new TrainSample[samples.length];
        for (int i = 0; i < samples.length; i++) {
            training[i] = new TrainSample();
            training[i].query = samples[i];
            training[i].neighbors = new ArrayList<>();
            ArrayList<Neighbor<double[], double[]>> neighbors = new ArrayList<>();
            range.range(training[i].query, radius, neighbors);
            for (Neighbor<double[], double[]> n : neighbors) {
                training[i].neighbors.add(keys.get(n.index));
            }
        }

        model = new ArrayList<>(hash.size());
        for (int i = 0; i < hash.size(); i++) {
            model.add(new PosterioriModel(hash.get(i), training, Nz, sigma));
        }
    }
788030
def iter_alignments(dataset, cognate_sets, column='Segments', method='library'):
    
    if not isinstance(dataset, lingpy.basic.parser.QLCParser):
        wordlist = _cldf2wordlist(dataset)
        cognates = {r['Form_ID']: r for r in cognate_sets}
        wordlist.add_entries(
            'cogid',
            'lid',
            lambda x: cognates[x]['Cognateset_ID'] if x in cognates else 0)
        alm = lingpy.Alignments(
            wordlist,
            ref='cogid',
            row='parameter_id',
            col='language_id',
            segments=column.lower())
        alm.align(method=method)
        for k in alm:
            if alm[k, 'lid'] in cognates:
                cognate = cognates[alm[k, 'lid']]
                cognate['Alignment'] = alm[k, 'alignment']
                cognate['Alignment_Method'] = method
    else:
        alm = lingpy.Alignments(dataset, ref='cogid')
        alm.align(method=method)

        for cognate in cognate_sets:
            idx = cognate['ID'] or cognate['Form_ID']
            cognate['Alignment'] = alm[int(idx), 'alignment']
            cognate['Alignment_Method'] = 'SCA-' + method
548609
def train_phrases(paths, out='data/bigram_model.phrases', tokenizer=word_tokenize, **kwargs):
    
    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    # Change to use less memory. Default is 40m.
    kwargs = {
        'max_vocab_size': 40000000,
        'threshold': 8.
    }.update(kwargs)

    print('Training bigrams...')
    bigram = Phrases(_phrase_doc_stream(paths, n, tokenizer=word_tokenize), **kwargs)

    print('Saving...')
    bigram.save(out)
780910
def generate_pos_tagger(check_accuracy=False):
    """"""
    global tagger
    logging.debug("Reading TIGER corpus")
    corp = nltk.corpus.ConllCorpusReader(DIR_PATH, TIGER_FILE_NAME,
                                         ['ignore', 'words', 'ignore', 'ignore', 'pos'],
                                         encoding='utf-8')
    tagged_sents = list(corp.tagged_sents())
    logging.debug("Shuffling sentences")
    random.shuffle(tagged_sents)
    if check_accuracy:
        # set a split size: use 90% for training, 10% for testing
        split_perc = 0.1
        split_size = int(len(tagged_sents) * split_perc)
        train_sents, test_sents = tagged_sents[split_size:], tagged_sents[:split_size]
    else:
        train_sents = tagged_sents
    logging.debug("Training Tagger")
    tagger = ClassifierBasedGermanTagger(train=train_sents)
    logging.debug("Training finished")
    if check_accuracy:
        accuracy = tagger.evaluate(test_sents)
        logging.debug("Accurracy is {}.".format(accuracy))
    logging.debug("Serializing the Tagger")
    with open(os.path.join(DIR_PATH, TAGGER_FILE_NAME), 'wb') as f:
        pickle.dump(tagger, f, protocol=3)
956916
def tabbed_parsing_token_generator(data_dir, tmp_dir, train, prefix,
                                   source_vocab_size, target_vocab_size):
  """"""
  filename = "parsing_{0}.pairs".format("train" if train else "dev")
  source_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 0,
      prefix + "_source.tokens.vocab.%d" % source_vocab_size, source_vocab_size)
  target_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 1,
      prefix + "_target.tokens.vocab.%d" % target_vocab_size, target_vocab_size)
  pair_filepath = os.path.join(tmp_dir, filename)
  return text_problems.text2text_generate_encoded(
      text_problems.text2text_txt_tab_iterator(pair_filepath), source_vocab,
      target_vocab)
463621
def get_train_data(args):
    """"""
    counter = dict()
    with io.open(args.vocab, 'r', encoding='utf-8') as f:
        for line in f:
            token, count = line.split('\t')
            counter[token] = int(count)
    vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None,
                      bos_token=None, eos_token=None, min_freq=1)

    npz = np.load(args.cooccurrences)
    row, col, counts = npz['row'], npz['col'], npz['data']

    rank_dtype = 'int32'
    if row.max() >= np.iinfo(np.int32).max:
        rank_dtype = 'int64'
        # MXNet has no support for uint32, so we must fall back to int64
        logging.info('More words than could be counted using int32. '
                     'Using int64 to represent word indices.')
    row = mx.nd.array(row, dtype=rank_dtype)
    col = mx.nd.array(col, dtype=rank_dtype)
    # row is always used as 'source' and col as 'context' word. Therefore
    # duplicate the entries.

    assert row.shape == col.shape
    row = mx.nd.concatenate([row, col])
    col = mx.nd.concatenate([col, row[:len(row) // 2]])

    counts = mx.nd.array(counts, dtype='float32')
    counts = mx.nd.concatenate([counts, counts])

    return vocab, row, col, counts
1270732
def create_examples_train(candidate_dialog_paths, rng, positive_probability=0.5, max_context_length=20):
    
    i = 0
    examples = []
    for context_dialog in candidate_dialog_paths:
        if i % 1000 == 0:
            print str(i)
        dialog_path = candidate_dialog_paths[i]
        examples.append(create_single_dialog_train_example(dialog_path, candidate_dialog_paths, rng, positive_probability,
                                                           max_context_length=max_context_length))
        i += 1
556300
def train(ctx, output, corpus, clusters):
    """"""
    click.echo('chemdataextractor.pos.train')
    click.echo('Output: %s' % output)
    click.echo('Corpus: %s' % corpus)
    click.echo('Clusters: %s' % clusters)

    wsj_sents = []
    genia_sents = []

    if corpus == 'wsj' or corpus == 'wsj+genia':
        wsj_sents = list(wsj_training.tagged_sents())
        # For WSJ, remove all tokens with -NONE- tag
        for i, wsj_sent in enumerate(wsj_sents):
            wsj_sents[i] = [t for t in wsj_sent if not t[1] == '-NONE-']

    if corpus == 'genia' or corpus == 'wsj+genia':
        genia_sents = list(genia_training.tagged_sents())
        # Translate GENIA
        for i, genia_sent in enumerate(genia_sents):
            for j, (token, tag) in enumerate(genia_sent):
                if tag == '(':
                    genia_sents[i][j] = (token, '-LRB-')  # ( to -RLB- (also do for evaluation)
                elif tag == ')':
                    genia_sents[i][j] = (token, '-RRB-')  # ) to -RRB- (also do for evaluation)
                elif tag == 'CT':
                    genia_sents[i][j] = (token, 'DT')  # Typo?
                elif tag == 'XT':
                    genia_sents[i][j] = (token, 'DT')  # Typo?
                elif tag == '-':
                    genia_sents[i][j] = (token, ':')  # Single hyphen character for dash
                elif tag == 'N':
                    genia_sents[i][j] = (token, 'NN')  # Typo?
                elif tag == 'PP':
                    genia_sents[i][j] = (token, 'PRP')  # Typo?
                elif tag == '' and token == ')':
                    genia_sents[i][j] = (token, '-RRB-')  # Typo?
                elif tag == '' and token == 'IFN-gamma':
                    genia_sents[i][j] = (token, 'NN')  # Typo?
                elif '|' in tag:
                    genia_sents[i][j] = (token, tag.split('|')[0])  # If contains |, choose first part
            # Filter any tags not in the allowed tagset (Shouldn't be any left anyway)
            genia_sents[i] = [t for t in genia_sent if t[1] in TAGS]

    if corpus == 'wsj':
        training_corpus = wsj_sents
    elif corpus == 'genia':
        training_corpus = genia_sents
    elif corpus == 'wsj+genia':
        training_corpus = wsj_sents + genia_sents
    else:
        raise click.ClickException('Invalid corpus')

    tagger = ChemCrfPosTagger(clusters=clusters)
    tagger.train(training_corpus, output)
691556
def tag(self, corpus, tokenize=True):
        ''''''
        # Assume untokenized corpus has \n between sentences and ' ' between words
        s_split = SentenceTokenizer().tokenize if tokenize else lambda t: t.split('\n')
        w_split = WordTokenizer().tokenize if tokenize else lambda s: s.split()
        def split_sents(corpus):
            for s in s_split(corpus):
                yield w_split(s)

        prev, prev2 = self.START
        tokens = []
        for words in split_sents(corpus):
            context = self.START + [self._normalize(w) for w in words] + self.END
            for i, word in enumerate(words):
                tag = self.tagdict.get(word)
                if not tag:
                    features = self._get_features(i, word, context, prev, prev2)
                    tag = self.model.predict(features)
                tokens.append((word, tag))
                prev2 = prev
                prev = tag
        return tokens