Index words char trie.

@param documents the documents
@param maxLevels the max levels
@param minWeight the min weight
@return the char trie
7712
public static CharTrie indexWords(Collection<String> documents, int maxLevels, int minWeight) {
    return create(documents, maxLevels, minWeight, true);
  }
343356
@SuppressWarnings("rawtypes")
	public Set zrangeByScore(Object key, double min, double max) {
		Jedis jedis = getJedis();
		try {
			Set<byte[]> data = jedis.zrangeByScore(keyToBytes(key), min, max);
			Set<Object> result = new LinkedHashSet<Object>();	// 有序集合必须 LinkedHashSet
			valueSetFromBytesSet(data, result);
			return result;
		}
		finally {close(jedis);}
	}
1138650
public static function text($chars, $min, $max)
    {
        $length = mt_rand($min, $max);
        $charLength = mb_strlen($chars);
        $result = '';
        for($i = 0; $i < $length; ++$i)
        {
            $result .= mb_substr($chars, mt_rand(1, $charLength) - 1, 1);
        }
        return $result;
    }
1186198
public static <TERM> Iterable<Map<TERM, Double>> tfs(Iterable<Collection<TERM>> documents, TfType type)
    {
        List<Map<TERM, Double>> tfs = new ArrayList<Map<TERM, Double>>();
        for (Collection<TERM> document : documents)
        {
            tfs.add(tf(document, type));
        }
        return tfs;
    }
1124425
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // prefetch the first windowSize words 
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i)
            nextWords.offer(documentTokens.next());
        
        while (!nextWords.isEmpty()) {
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);
            
            if (calculateSemantics) {
                IntegerVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    add(focusMeaning, iv);
                }
            
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }

                    add(focusMeaning, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);
            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
1285472
public static FullTextIndex fullTextIndex(@NonNull FullTextIndexItem... items) {
        if (items == null) {
            throw new IllegalArgumentException("items cannot be null.");
        }
        return new FullTextIndex(items);
    }
676287
public static <TERM> Iterable<Map<TERM, Double>> tfs(Iterable<Collection<TERM>> documents)
    {
        return tfs(documents, TfType.NATURAL);
    }
1026278
public Collection<String> nearestLabels(@NonNull String rawText, int topN) {
        List<String> tokens = tokenizerFactory.create(rawText).getTokens();
        List<VocabWord> document = new ArrayList<>();
        for (String token : tokens) {
            if (vocab.containsWord(token)) {
                document.add(vocab.wordFor(token));
            }
        }

        // we're returning empty collection for empty document
        if (document.isEmpty()) {
            log.info("Document passed to nearestLabels() has no matches in model vocabulary");
            return new ArrayList<>();
        }

        return nearestLabels(document, topN);
    }
67401
public Collection<String> nearestLabels(LabelledDocument document, int topN) {
        if (document.getReferencedContent() != null) {
            return nearestLabels(document.getReferencedContent(), topN);
        } else
            return nearestLabels(document.getContent(), topN);
    }
1250471
public Set<String> getUniqueWords(Object document) {
        Set<String> features = new FastSet<String>();
		if (document instanceof AnalyzedText){
	        features.addAll(((AnalyzedText)document).getUniqueWords());
		    if (includeLengthCategory){
		       	features.add(((AnalyzedText)document).getLengthCategory().toString());
		    }
		}else{
	        addWords(document, features);
		}
	    return features;
	}
613784
protected void createDefaultLevels() {
        this.listLevels = new ArrayList();	// listlevels
        for(int i=0; i<=8; i++) {
            // create a list level
            RtfListLevel ll = new RtfListLevel(this.document);
            ll.setListType(RtfListLevel.LIST_TYPE_NUMBERED);
        	ll.setFirstIndent(0);
        	ll.setLeftIndent(0);
        	ll.setLevelTextNumber(i);
            ll.setTentative(true);
            ll.correctIndentation();
            this.listLevels.add(ll);
        }

    }
649369
@Override
    public Collection<String> wordsNearest(INDArray words, int top) {
        if (lookupTable instanceof InMemoryLookupTable) {
            InMemoryLookupTable l = (InMemoryLookupTable) lookupTable;

            INDArray syn0 = l.getSyn0();

            if (!normalized) {
                synchronized (this) {
                    if (!normalized) {
                        syn0.diviColumnVector(syn0.norm2(1));
                        normalized = true;
                    }
                }
            }

            INDArray similarity = Transforms.unitVec(words).mmul(syn0.transpose());

            List<Double> highToLowSimList = getTopN(similarity, top + 20);

            List<WordSimilarity> result = new ArrayList<>();

            for (int i = 0; i < highToLowSimList.size(); i++) {
                String word = vocabCache.wordAtIndex(highToLowSimList.get(i).intValue());
                if (word != null && !word.equals("UNK") && !word.equals("STOP")) {
                    INDArray otherVec = lookupTable.vector(word);
                    double sim = Transforms.cosineSim(words, otherVec);

                    result.add(new WordSimilarity(word, sim));
                }
            }

            Collections.sort(result, new SimilarityComparator());

            return getLabels(result, top);
        }

        Counter<String> distances = new Counter<>();

        for (String s : vocabCache.words()) {
            INDArray otherVec = lookupTable.vector(s);
            double sim = Transforms.cosineSim(words, otherVec);
            distances.incrementCount(s, (float) sim);
        }


        distances.keepTopNElements(top);
        return distances.keySet();


    }
34060
public List<INDArray> inferVectorBatched(@NonNull List<String> documents) {
        if (countSubmitted == null)
            initInference();

        if (this.vocab == null || this.vocab.numWords() == 0)
            reassignExistingModel();

        List<Future<INDArray>> futuresList = new ArrayList<>();
        List<INDArray> results = new ArrayList<>();

        final AtomicLong flag = new AtomicLong(0);

        for (int i = 0; i < documents.size(); i++) {
            BlindInferenceCallable callable =
                            new BlindInferenceCallable(vocab, tokenizerFactory, documents.get(i), flag);

            futuresList.add(inferenceExecutor.submit(callable));
        }

        for (int i = 0; i < documents.size(); i++) {
            Future<INDArray> future = futuresList.get(i);
            try {
                results.add(future.get());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(e);
            } catch (ExecutionException e) {
                throw new RuntimeException(e);
            }
        }

        return results;
    }
1285963
def get_word_index(tokens, char_index):
    '''
    
    '''
    for (i, token) in enumerate(tokens):
        if token['char_end'] == 0:
            continue
        if token['char_begin'] <= char_index and char_index <= token['char_end']:
            return i
    return 0
909451
public void getChars(int start, int end, char[] dest, int destoff) {
        checkRange("getChars", start, end);

        if (end <= mGapStart) {
            System.arraycopy(mText, start, dest, destoff, end - start);
        } else if (start >= mGapStart) {
            System.arraycopy(mText, start + mGapLength, dest, destoff, end - start);
        } else {
            System.arraycopy(mText, start, dest, destoff, mGapStart - start);
            System.arraycopy(mText, mGapStart + mGapLength,
                    dest, destoff + (mGapStart - start),
                    end - mGapStart);
        }
    }
240824
public LinkedList<Map.Entry<String, V>> commonPrefixSearchWithValue(char[] chars, int begin)
    {
        LinkedList<Map.Entry<String, V>> result = new LinkedList<Map.Entry<String, V>>();
        StringBuilder sb = new StringBuilder();
        BaseNode branch = this;
        for (int i = begin; i < chars.length; ++i)
        {
            char aChar = chars[i];
            branch = branch.getChild(aChar);
            if (branch == null || branch.status == Status.UNDEFINED_0) return result;
            sb.append(aChar);
            if (branch.status == Status.WORD_MIDDLE_2 || branch.status == Status.WORD_END_3)
            {
                result.add(new AbstractMap.SimpleEntry<String, V>(sb.toString(), (V) branch.value));
            }
        }

        return result;
    }
917764
public static Pattern atLeast(final int min, final CharPredicate predicate) {
    Checks.checkMin(min);
    return new Pattern() {
      @Override public int match(CharSequence src, int begin, int end) {
        int minLen = RepeatCharPredicatePattern.matchRepeat(min, predicate, src, end, begin, 0);
        if (minLen == MISMATCH) return MISMATCH;
        return matchMany(predicate, src, end, begin + minLen, minLen);
      }
      @Override public String toString() {
        return (min > 1) ? (predicate + "{" + min + ",}") : (predicate + "+");
      }
    };
  }