Train on the corpus
@param rdd the rdd to train
@return the vocab and weights
502120
public Pair<VocabCache<VocabWord>, GloveWeightLookupTable> train(JavaRDD<String> rdd) throws Exception {
        // Each `train()` can use different parameters
        final JavaSparkContext sc = new JavaSparkContext(rdd.context());
        final SparkConf conf = sc.getConf();
        final int vectorLength = assignVar(VECTOR_LENGTH, conf, Integer.class);
        final boolean useAdaGrad = assignVar(ADAGRAD, conf, Boolean.class);
        final double negative = assignVar(NEGATIVE, conf, Double.class);
        final int numWords = assignVar(NUM_WORDS, conf, Integer.class);
        final int window = assignVar(WINDOW, conf, Integer.class);
        final double alpha = assignVar(ALPHA, conf, Double.class);
        final double minAlpha = assignVar(MIN_ALPHA, conf, Double.class);
        final int iterations = assignVar(ITERATIONS, conf, Integer.class);
        final int nGrams = assignVar(N_GRAMS, conf, Integer.class);
        final String tokenizer = assignVar(TOKENIZER, conf, String.class);
        final String tokenPreprocessor = assignVar(TOKEN_PREPROCESSOR, conf, String.class);
        final boolean removeStop = assignVar(REMOVE_STOPWORDS, conf, Boolean.class);

        Map<String, Object> tokenizerVarMap = new HashMap<String, Object>() {
            {
                put("numWords", numWords);
                put("nGrams", nGrams);
                put("tokenizer", tokenizer);
                put("tokenPreprocessor", tokenPreprocessor);
                put("removeStop", removeStop);
            }
        };
        Broadcast<Map<String, Object>> broadcastTokenizerVarMap = sc.broadcast(tokenizerVarMap);


        TextPipeline pipeline = new TextPipeline(rdd, broadcastTokenizerVarMap);
        pipeline.buildVocabCache();
        pipeline.buildVocabWordListRDD();


        // Get total word count
        Long totalWordCount = pipeline.getTotalWordCount();
        VocabCache<VocabWord> vocabCache = pipeline.getVocabCache();
        JavaRDD<Pair<List<String>, AtomicLong>> sentenceWordsCountRDD = pipeline.getSentenceWordsCountRDD();
        final Pair<VocabCache<VocabWord>, Long> vocabAndNumWords = new Pair<>(vocabCache, totalWordCount);

        vocabCacheBroadcast = sc.broadcast(vocabAndNumWords.getFirst());

        final GloveWeightLookupTable gloveWeightLookupTable = new GloveWeightLookupTable.Builder()
                        .cache(vocabAndNumWords.getFirst()).lr(conf.getDouble(GlovePerformer.ALPHA, 0.01))
                        .maxCount(conf.getDouble(GlovePerformer.MAX_COUNT, 100))
                        .vectorLength(conf.getInt(GlovePerformer.VECTOR_LENGTH, 300))
                        .xMax(conf.getDouble(GlovePerformer.X_MAX, 0.75)).build();
        gloveWeightLookupTable.resetWeights();

        gloveWeightLookupTable.getBiasAdaGrad().historicalGradient = Nd4j.ones(gloveWeightLookupTable.getSyn0().rows());
        gloveWeightLookupTable.getWeightAdaGrad().historicalGradient =
                        Nd4j.ones(gloveWeightLookupTable.getSyn0().shape());


        log.info("Created lookup table of size " + Arrays.toString(gloveWeightLookupTable.getSyn0().shape()));
        CounterMap<String, String> coOccurrenceCounts = sentenceWordsCountRDD
                        .map(new CoOccurrenceCalculator(symmetric, vocabCacheBroadcast, windowSize))
                        .fold(new CounterMap<String, String>(), new CoOccurrenceCounts());
        Iterator<Pair<String, String>> pair2 = coOccurrenceCounts.getIterator();
        List<Triple<String, String, Float>> counts = new ArrayList<>();

        while (pair2.hasNext()) {
            Pair<String, String> next = pair2.next();
            if (coOccurrenceCounts.getCount(next.getFirst(), next.getSecond()) > gloveWeightLookupTable.getMaxCount()) {
                coOccurrenceCounts.setCount(next.getFirst(), next.getSecond(),
                                (float) gloveWeightLookupTable.getMaxCount());
            }
            counts.add(new Triple<>(next.getFirst(), next.getSecond(),
                    (float) coOccurrenceCounts.getCount(next.getFirst(), next.getSecond())));

        }

        log.info("Calculated co occurrences");

        JavaRDD<Triple<String, String, Float>> parallel = sc.parallelize(counts);
        JavaPairRDD<String, Tuple2<String, Float>> pairs = parallel
                        .mapToPair(new PairFunction<Triple<String, String, Float>, String, Tuple2<String, Float>>() {
                            @Override
                            public Tuple2<String, Tuple2<String, Float>> call(
                                            Triple<String, String, Float> stringStringDoubleTriple) throws Exception {
                                return new Tuple2<>(stringStringDoubleTriple.getFirst(),
                                                new Tuple2<>(stringStringDoubleTriple.getSecond(),
                                                                stringStringDoubleTriple.getThird()));
                            }
                        });

        JavaPairRDD<VocabWord, Tuple2<VocabWord, Float>> pairsVocab = pairs.mapToPair(
                        new PairFunction<Tuple2<String, Tuple2<String, Float>>, VocabWord, Tuple2<VocabWord, Float>>() {
                            @Override
                            public Tuple2<VocabWord, Tuple2<VocabWord, Float>> call(
                                            Tuple2<String, Tuple2<String, Float>> stringTuple2Tuple2) throws Exception {
                                VocabWord w1 = vocabCacheBroadcast.getValue().wordFor(stringTuple2Tuple2._1());
                                VocabWord w2 = vocabCacheBroadcast.getValue().wordFor(stringTuple2Tuple2._2()._1());
                                return new Tuple2<>(w1, new Tuple2<>(w2, stringTuple2Tuple2._2()._2()));
                            }
                        });


        for (int i = 0; i < iterations; i++) {
            JavaRDD<GloveChange> change =
                            pairsVocab.map(new Function<Tuple2<VocabWord, Tuple2<VocabWord, Float>>, GloveChange>() {
                                @Override
                                public GloveChange call(
                                                Tuple2<VocabWord, Tuple2<VocabWord, Float>> vocabWordTuple2Tuple2)
                                                throws Exception {
                                    VocabWord w1 = vocabWordTuple2Tuple2._1();
                                    VocabWord w2 = vocabWordTuple2Tuple2._2()._1();
                                    INDArray w1Vector = gloveWeightLookupTable.getSyn0().slice(w1.getIndex());
                                    INDArray w2Vector = gloveWeightLookupTable.getSyn0().slice(w2.getIndex());
                                    INDArray bias = gloveWeightLookupTable.getBias();
                                    double score = vocabWordTuple2Tuple2._2()._2();
                                    double xMax = gloveWeightLookupTable.getxMax();
                                    double maxCount = gloveWeightLookupTable.getMaxCount();
                                    //w1 * w2 + bias
                                    double prediction = Nd4j.getBlasWrapper().dot(w1Vector, w2Vector);
                                    prediction += bias.getDouble(w1.getIndex()) + bias.getDouble(w2.getIndex());

                                    double weight = FastMath.pow(Math.min(1.0, (score / maxCount)), xMax);

                                    double fDiff = score > xMax ? prediction : weight * (prediction - Math.log(score));
                                    if (Double.isNaN(fDiff))
                                        fDiff = Nd4j.EPS_THRESHOLD;
                                    //amount of change
                                    double gradient = fDiff;

                                    Pair<INDArray, Float> w1Update = update(gloveWeightLookupTable.getWeightAdaGrad(),
                                                    gloveWeightLookupTable.getBiasAdaGrad(),
                                                    gloveWeightLookupTable.getSyn0(), gloveWeightLookupTable.getBias(),
                                                    w1, w1Vector, w2Vector, gradient);
                                    Pair<INDArray, Float> w2Update = update(gloveWeightLookupTable.getWeightAdaGrad(),
                                                    gloveWeightLookupTable.getBiasAdaGrad(),
                                                    gloveWeightLookupTable.getSyn0(), gloveWeightLookupTable.getBias(),
                                                    w2, w2Vector, w1Vector, gradient);
                                    return new GloveChange(w1, w2, w1Update.getFirst(), w2Update.getFirst(),
                                                    w1Update.getSecond(), w2Update.getSecond(), fDiff,
                                                    gloveWeightLookupTable.getWeightAdaGrad().getHistoricalGradient()
                                                                    .slice(w1.getIndex()),
                                                    gloveWeightLookupTable.getWeightAdaGrad().getHistoricalGradient()
                                                                    .slice(w2.getIndex()),
                                                    gloveWeightLookupTable.getBiasAdaGrad().getHistoricalGradient()
                                                                    .getDouble(w2.getIndex()),
                                                    gloveWeightLookupTable.getBiasAdaGrad().getHistoricalGradient()
                                                                    .getDouble(w1.getIndex()));

                                }
                            });



            List<GloveChange> gloveChanges = change.collect();
            double error = 0.0;
            for (GloveChange change2 : gloveChanges) {
                change2.apply(gloveWeightLookupTable);
                error += change2.getError();
            }


            List l = pairsVocab.collect();
            Collections.shuffle(l);
            pairsVocab = sc.parallelizePairs(l);

            log.info("Error at iteration " + i + " was " + error);



        }

        return new Pair<>(vocabAndNumWords.getFirst(), gloveWeightLookupTable);
    }
213983
def load_dataset(data_name):
    """"""
    if data_name == 'MR' or data_name == 'Subj':
        train_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, [])
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths
    else:
        train_dataset, test_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, test_dataset)
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        test_dataset, test_data_lengths = _preprocess_dataset(test_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths, test_dataset, \
               test_data_lengths
647834
def fit(self, dataset):
        
        if not isinstance(dataset, RDD):
            raise TypeError("dataset should be an RDD of term frequency vectors")
        jmodel = callMLlibFunc("fitIDF", self.minDocFreq, dataset.map(_convert_to_vector))
        return IDFModel(jmodel)
1083426
def _build_vocab(self, sentences, markers=[]):
        
        from snorkel.learning.pytorch.rnn.utils import SymbolTable

        vocab = Counter()
        for sent in sentences:
            for w in sent:
                vocab[w] += 1
        word_dict = SymbolTable()
        list(map(word_dict.get, vocab))
        list(map(word_dict.get, markers))
        return word_dict
1026278
public Collection<String> nearestLabels(@NonNull String rawText, int topN) {
        List<String> tokens = tokenizerFactory.create(rawText).getTokens();
        List<VocabWord> document = new ArrayList<>();
        for (String token : tokens) {
            if (vocab.containsWord(token)) {
                document.add(vocab.wordFor(token));
            }
        }

        // we're returning empty collection for empty document
        if (document.isEmpty()) {
            log.info("Document passed to nearestLabels() has no matches in model vocabulary");
            return new ArrayList<>();
        }

        return nearestLabels(document, topN);
    }
993377
public static GloveWeightLookupTable load(InputStream is, VocabCache<? extends SequenceElement> vocab)
                    throws IOException {
        LineIterator iter = IOUtils.lineIterator(is, "UTF-8");
        GloveWeightLookupTable glove = null;
        Map<String, float[]> wordVectors = new HashMap<>();
        while (iter.hasNext()) {
            String line = iter.nextLine().trim();
            if (line.isEmpty())
                continue;
            String[] split = line.split(" ");
            String word = split[0];
            if (glove == null)
                glove = new GloveWeightLookupTable.Builder().cache(vocab).vectorLength(split.length - 1).build();



            if (word.isEmpty())
                continue;
            float[] read = read(split, glove.layerSize());
            if (read.length < 1)
                continue;


            wordVectors.put(word, read);



        }

        glove.setSyn0(weights(glove, wordVectors, vocab));
        glove.resetWeights(false);


        iter.close();


        return glove;

    }
787913
def create_training_instances(x):
    """"""
    (input_files, out, tokenizer, max_seq_length, dupe_factor,
     short_seq_prob, masked_lm_prob, max_predictions_per_seq, rng) = x
    time_start = time.time()
    logging.info('Processing %s', input_files)
    all_documents = [[]]

    # Input file format:
    # (1) One sentence per line. These should ideally be actual sentences, not
    # entire paragraphs or arbitrary spans of text. (Because we use the
    # sentence boundaries for the "next sentence prediction" task).
    # (2) Blank lines between documents. Document boundaries are needed so
    # that the "next sentence prediction" task doesn't span between documents.
    for input_file in input_files:
        with io.open(input_file, 'r', encoding='UTF-8') as reader:
            while True:
                line = reader.readline()
                if not line:
                    break
                line = line.strip()

                # Empty lines are used as document delimiters
                if not line:
                    all_documents.append([])
                tokens = tokenizer(line) if not args.tokenized else line.split(' ')
                if tokens:
                    all_documents[-1].append(tokens)

    # Remove empty documents
    all_documents = [x for x in all_documents if x]
    rng.shuffle(all_documents)

    vocab_words = tokenizer.vocab.idx_to_token
    instances = []
    for _ in range(dupe_factor):
        for document_index in range(len(all_documents)):
            instances.extend(
                create_instances_from_document(
                    all_documents, document_index, max_seq_length,
                    short_seq_prob, masked_lm_prob, max_predictions_per_seq,
                    vocab_words, rng))

    rng.shuffle(instances)
    input_ids = []
    segment_ids = []
    masked_lm_positions = []
    masked_lm_ids = []
    masked_lm_weights = []
    next_sentence_labels = []
    valid_lengths = []

    for inst_index, instance in enumerate(instances):
        feature = transform(instance, tokenizer, max_seq_length, max_predictions_per_seq, False)
        input_ids.append(
            np.ascontiguousarray(feature['input_ids'], dtype='int32'))
        segment_ids.append(
            np.ascontiguousarray(feature['segment_ids'], dtype='int32'))
        masked_lm_positions.append(
            np.ascontiguousarray(feature['masked_lm_positions'], dtype='int32'))
        masked_lm_ids.append(np.ascontiguousarray(feature['masked_lm_ids'], dtype='int32'))
        masked_lm_weights.append(
            np.ascontiguousarray(feature['masked_lm_weights'], dtype='float32'))
        next_sentence_labels.append(feature['next_sentence_labels'][0])
        valid_lengths.append(feature['valid_lengths'][0])
        if inst_index < 20:
            print_example(instance, feature)

    features = (input_ids, segment_ids, masked_lm_positions, masked_lm_ids, \
                masked_lm_weights, next_sentence_labels, valid_lengths)

    logging.info('*** Writing to output file %s ***', out)
    if args.format == 'numpy':
        write_to_files_np(features, tokenizer, args.max_seq_length,
                          args.max_predictions_per_seq, [out])
    elif args.format == 'recordio':
        write_to_files_rec(instances, tokenizer, args.max_seq_length,
                           args.max_predictions_per_seq, [out])
    else:
        raise ValueError('unsupported format: %s'%args.format)

    time_end = time.time()
    logging.info('Process %d files took %.1f s', len(input_files), time_end - time_start)
548609
def train_phrases(paths, out='data/bigram_model.phrases', tokenizer=word_tokenize, **kwargs):
    
    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    # Change to use less memory. Default is 40m.
    kwargs = {
        'max_vocab_size': 40000000,
        'threshold': 8.
    }.update(kwargs)

    print('Training bigrams...')
    bigram = Phrases(_phrase_doc_stream(paths, n, tokenizer=word_tokenize), **kwargs)

    print('Saving...')
    bigram.save(out)
1372092
public static Vocab of(Vocab... vocabs)
  {
    return new AggregateVocab(new ImmutableList.Builder<Vocab>().add(vocabs).build());
  }
88266
def _load(cls,
              config: Params,
              serialization_dir: str,
              weights_file: str = None,
              cuda_device: int = -1) -> 'Model':
        
        weights_file = weights_file or os.path.join(serialization_dir, _DEFAULT_WEIGHTS)

        # Load vocabulary from file
        vocab_dir = os.path.join(serialization_dir, 'vocabulary')
        # If the config specifies a vocabulary subclass, we need to use it.
        vocab_params = config.get("vocabulary", Params({}))
        vocab_choice = vocab_params.pop_choice("type", Vocabulary.list_available(), True)
        vocab = Vocabulary.by_name(vocab_choice).from_files(vocab_dir)

        model_params = config.get('model')

        # The experiment config tells us how to _train_ a model, including where to get pre-trained
        # embeddings from.  We're now _loading_ the model, so those embeddings will already be
        # stored in our weights.  We don't need any pretrained weight file anymore, and we don't
        # want the code to look for it, so we remove it from the parameters here.
        remove_pretrained_embedding_params(model_params)
        model = Model.from_params(vocab=vocab, params=model_params)

        # If vocab+embedding extension was done, the model initialized from from_params
        # and one defined by state dict in weights_file might not have same embedding shapes.
        # Eg. when model embedder module was transferred along with vocab extension, the
        # initialized embedding weight shape would be smaller than one in the state_dict.
        # So calling model embedding extension is required before load_state_dict.
        # If vocab and model embeddings are in sync, following would be just a no-op.
        model.extend_embedder_vocab()

        model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device))
        model.load_state_dict(model_state)

        # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are
        # in sync with the weights
        if cuda_device >= 0:
            model.cuda(cuda_device)
        else:
            model.cpu()

        return model
1254702
def train(args):
    """"""
    if not args.model.lower() in ['cbow', 'skipgram']:
        logging.error('Unsupported model %s.', args.model)
        sys.exit(1)

    if args.data.lower() == 'toy':
        data = mx.gluon.data.SimpleDataset(nlp.data.Text8(segment='train')[:2])
        data, vocab, idx_to_counts = preprocess_dataset(
            data, max_vocab_size=args.max_vocab_size)
    elif args.data.lower() == 'text8':
        data = nlp.data.Text8(segment='train')
        data, vocab, idx_to_counts = preprocess_dataset(
            data, max_vocab_size=args.max_vocab_size)
    elif args.data.lower() == 'fil9':
        data = nlp.data.Fil9(max_sentence_length=10000)
        data, vocab, idx_to_counts = preprocess_dataset(
            data, max_vocab_size=args.max_vocab_size)
    elif args.data.lower() == 'wiki':
        data, vocab, idx_to_counts = wiki(args.wiki_root, args.wiki_date,
                                          args.wiki_language,
                                          args.max_vocab_size)

    if args.ngram_buckets > 0:
        data, batchify_fn, subword_function = transform_data_fasttext(
            data, vocab, idx_to_counts, cbow=args.model.lower() == 'cbow',
            ngram_buckets=args.ngram_buckets, ngrams=args.ngrams,
            batch_size=args.batch_size, window_size=args.window,
            frequent_token_subsampling=args.frequent_token_subsampling)
    else:
        subword_function = None
        data, batchify_fn = transform_data_word2vec(
            data, vocab, idx_to_counts, cbow=args.model.lower() == 'cbow',
            batch_size=args.batch_size, window_size=args.window,
            frequent_token_subsampling=args.frequent_token_subsampling)

    num_tokens = float(sum(idx_to_counts))

    model = CBOW if args.model.lower() == 'cbow' else SG
    embedding = model(token_to_idx=vocab.token_to_idx, output_dim=args.emsize,
                      batch_size=args.batch_size, num_negatives=args.negative,
                      negatives_weights=mx.nd.array(idx_to_counts),
                      subword_function=subword_function)
    context = get_context(args)
    embedding.initialize(ctx=context)
    if not args.no_hybridize:
        embedding.hybridize(static_alloc=True, static_shape=True)

    optimizer_kwargs = dict(learning_rate=args.lr)
    try:
        trainer = mx.gluon.Trainer(embedding.collect_params(), args.optimizer,
                                   optimizer_kwargs)
    except ValueError as e:
        if args.optimizer == 'groupadagrad':
            logging.warning('MXNet <= v1.3 does not contain '
                            'GroupAdaGrad support. Falling back to AdaGrad')
            trainer = mx.gluon.Trainer(embedding.collect_params(), 'adagrad',
                                       optimizer_kwargs)
        else:
            raise e

    try:
        if args.no_prefetch_batch:
            data = data.transform(batchify_fn)
        else:
            from executors import LazyThreadPoolExecutor
            num_cpu = len(os.sched_getaffinity(0))
            ex = LazyThreadPoolExecutor(num_cpu)
    except (ImportError, SyntaxError, AttributeError):
        # Py2 - no async prefetching is supported
        logging.warning(
            'Asynchronous batch prefetching is not supported on Python 2. '
            'Consider upgrading to Python 3 for improved performance.')
        data = data.transform(batchify_fn)

    num_update = 0
    prefetched_iters = []
    for _ in range(min(args.num_prefetch_epoch, args.epochs)):
        prefetched_iters.append(iter(data))
    for epoch in range(args.epochs):
        if epoch + len(prefetched_iters) < args.epochs:
            prefetched_iters.append(iter(data))
        data_iter = prefetched_iters.pop(0)
        try:
            batches = ex.map(batchify_fn, data_iter)
        except NameError:  # Py 2 or batch prefetching disabled
            batches = data_iter

        # Logging variables
        log_wc = 0
        log_start_time = time.time()
        log_avg_loss = 0

        for i, batch in enumerate(batches):
            ctx = context[i % len(context)]
            batch = [array.as_in_context(ctx) for array in batch]
            with mx.autograd.record():
                loss = embedding(*batch)
            loss.backward()

            num_update += loss.shape[0]
            if len(context) == 1 or (i + 1) % len(context) == 0:
                trainer.step(batch_size=1)

            # Logging
            log_wc += loss.shape[0]
            log_avg_loss += loss.mean().as_in_context(context[0])
            if (i + 1) % args.log_interval == 0:
                # Forces waiting for computation by computing loss value
                log_avg_loss = log_avg_loss.asscalar() / args.log_interval
                wps = log_wc / (time.time() - log_start_time)
                # Due to subsampling, the overall number of batches is an upper
                # bound
                num_batches = num_tokens // args.batch_size
                if args.model.lower() == 'skipgram':
                    num_batches = (num_tokens * args.window * 2) // args.batch_size
                else:
                    num_batches = num_tokens // args.batch_size
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'.format(
                                 epoch, i + 1, num_batches, log_avg_loss,
                                 wps / 1000, log_wc / 1000))
                log_start_time = time.time()
                log_avg_loss = 0
                log_wc = 0

            if args.eval_interval and (i + 1) % args.eval_interval == 0:
                with print_time('mx.nd.waitall()'):
                    mx.nd.waitall()
                with print_time('evaluate'):
                    evaluate(args, embedding, vocab, num_update)

    # Evaluate
    with print_time('mx.nd.waitall()'):
        mx.nd.waitall()
    with print_time('evaluate'):
        evaluate(args, embedding, vocab, num_update,
                 eval_analogy=not args.no_eval_analogy)

    # Save params
    with print_time('save parameters'):
        embedding.save_parameters(os.path.join(args.logdir, 'embedding.params'))
242533
public ComputationGraph fit(JavaRDD<DataSet> rdd) {
        if (Nd4j.getExecutioner() instanceof GridExecutioner)
            ((GridExecutioner) Nd4j.getExecutioner()).flushQueue();

        trainingMaster.executeTraining(this, rdd);
        network.incrementEpochCount();
        return network;
    }
1169753
def get_data_iters_and_vocabs(args: argparse.Namespace,
                              model_folder: Optional[str]) -> Tuple['data_io.BaseParallelSampleIter',
                                                                    List[vocab.Vocab], vocab.Vocab, model.ModelConfig]:
    

    model_config = model.SockeyeModel.load_config(os.path.join(args.model, C.CONFIG_NAME))

    if args.max_seq_len is None:
        max_seq_len_source = model_config.config_data.max_seq_len_source
        max_seq_len_target = model_config.config_data.max_seq_len_target
    else:
        max_seq_len_source, max_seq_len_target = args.max_seq_len

    batch_num_devices = 1 if args.use_cpu else sum(-di if di < 0 else 1 for di in args.device_ids)

    # Load the existing vocabs created when starting the training run.
    source_vocabs = vocab.load_source_vocabs(model_folder)
    target_vocab = vocab.load_target_vocab(model_folder)

    sources = [args.source] + args.source_factors
    sources = [str(os.path.abspath(source)) for source in sources]

    score_iter = data_io.get_scoring_data_iters(
        sources=sources,
        target=os.path.abspath(args.target),
        source_vocabs=source_vocabs,
        target_vocab=target_vocab,
        batch_size=args.batch_size,
        batch_num_devices=batch_num_devices,
        max_seq_len_source=max_seq_len_source,
        max_seq_len_target=max_seq_len_target)

    return score_iter, source_vocabs, target_vocab, model_config
725541
public Collection<String> predictSeveral(String rawText, int limit) {
        if (tokenizerFactory == null)
            throw new IllegalStateException("TokenizerFactory should be defined, prior to predict() call");

        List<String> tokens = tokenizerFactory.create(rawText).getTokens();
        List<VocabWord> document = new ArrayList<>();
        for (String token : tokens) {
            if (vocab.containsWord(token)) {
                document.add(vocab.wordFor(token));
            }
        }

        return predictSeveral(document, limit);
    }
880641
def _to_java_object_rdd(self):
        
        rdd = self._pickled()
        return self.ctx._jvm.SerDeUtil.pythonToJava(rdd._jrdd, True)
994337
public RDD textFile(ExtractorConfig<Cells> config) throws IllegalArgumentException {
        if(ExtractorConstants.HDFS.equals(config.getExtractorImplClassName())) {
            return createHDFSRDD(config);
        } else if(ExtractorConstants.S3.equals(config.getExtractorImplClassName())) {
            return createS3RDD(config);
        }
        throw new IllegalArgumentException("Valid configurations are HDFS paths, S3 paths or local file paths.");
    }
1038545
@Override
    public void trainOnInstance(Instance inst) {
        if (!this.isInit) {
            this.isInit = true;
            InstancesHeader instances = new InstancesHeader(dataset);
            this.learner.setModelContext(instances);
            this.learner.prepareForUse();
        }
        if (inst.weight() > 0) {
            inst.setDataset(dataset);
            learner.trainOnInstance(inst);
        }
    }