get the documents order by relevance score.

@param   sentence
@param   senWords
@throws  IOException
343436
protected Document[] textRankSortedDocuments(
            List<Sentence> sentence, List<List<IWord>> senWords) throws IOException
    {
        int docNum = sentence.size();
    
        //documents relevance matrix build
        double[][] relevance = BM25RelevanceMatixBuild(sentence, senWords);
        //org.lionsoul.jcseg.util.Util.printMatrix(relevance);
        
        double[] score = new double[docNum];
        double[] weight_sum = new double[docNum];
        for ( int i = 0; i < docNum; i++ ) {
            weight_sum[i] = sum(relevance[i]) - relevance[i][i];
            score[i] = 0;
        }
        
        //do the textrank score iteration
        for ( int c = 0; c < maxIterateNum; c++ ) {
            for ( int i = 0; i < docNum; i++ ) {
                double sigema = 0D;
                for ( int j = 0; j < docNum; j++ ) {
                    if ( i == j || weight_sum[j] == 0 ) continue;
                    /*
                     * ws(vj) * wji / sigma(wjk) with k in Out(Vj) 
                     * ws(vj): score[j] the score of document[j]
                     * wji: relevance score bettween document[j] and document[i]
                     * sigema(wjk): weight sum for document[j]
                    */
                    sigema += relevance[j][i] / weight_sum[j] * score[j];
                }
                
                score[i] = 1 - D + D * sigema;
            }
        }
        
        //build the document set
        //and sort the documents by scores
        Document[] docs = new Document[docNum];
        for ( int i = 0; i < docNum; i++ ) {
            docs[i] = new Document(i, sentence.get(i), senWords.get(i), score[i]);
        }
        
        Sort.shellSort(docs);
        
        
        //let gc do its works
        relevance = null;
        score = null;
        weight_sum = null;
        
        return docs;
    }
1124425
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // prefetch the first windowSize words 
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i)
            nextWords.offer(documentTokens.next());
        
        while (!nextWords.isEmpty()) {
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);
            
            if (calculateSemantics) {
                IntegerVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    add(focusMeaning, iv);
                }
            
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }

                    add(focusMeaning, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);
            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
191546
def _intertext_score(full_text):
    ''''''

    sentences = sentence_tokenizer(full_text)
    norm = _normalize(sentences)
    similarity_matrix = pairwise_kernels(norm, metric='cosine')
    scores = _textrank(similarity_matrix)
    scored_sentences = []
    for i, s in enumerate(sentences):
        scored_sentences.append((scores[i],i,s))
    top_scorers = sorted(scored_sentences,
                         key=lambda tup: tup[0], 
                         reverse=True)
    return top_scorers
254154
public INDArray loadSingleSentence(String sentence) {
        List<String> tokens = tokenizeSentence(sentence);
        if(format == Format.CNN1D || format == Format.RNN){
            int[] featuresShape = new int[] {1, wordVectorSize, Math.min(maxSentenceLength, tokens.size())};
            INDArray features = Nd4j.create(featuresShape, (format == Format.CNN1D ? 'c' : 'f'));
            INDArrayIndex[] indices = new INDArrayIndex[3];
            indices[0] = NDArrayIndex.point(0);
            for (int i = 0; i < featuresShape[2]; i++) {
                INDArray vector = getVector(tokens.get(i));
                indices[1] = NDArrayIndex.all();
                indices[2] = NDArrayIndex.point(i);
                features.put(indices, vector);
            }
            return features;
        } else {
            int[] featuresShape = new int[] {1, 1, 0, 0};
            if (sentencesAlongHeight) {
                featuresShape[2] = Math.min(maxSentenceLength, tokens.size());
                featuresShape[3] = wordVectorSize;
            } else {
                featuresShape[2] = wordVectorSize;
                featuresShape[3] = Math.min(maxSentenceLength, tokens.size());
            }

            INDArray features = Nd4j.create(featuresShape);
            int length = (sentencesAlongHeight ? featuresShape[2] : featuresShape[3]);
            INDArrayIndex[] indices = new INDArrayIndex[4];
            indices[0] = NDArrayIndex.point(0);
            indices[1] = NDArrayIndex.point(0);
            for (int i = 0; i < length; i++) {
                INDArray vector = getVector(tokens.get(i));

                if (sentencesAlongHeight) {
                    indices[2] = NDArrayIndex.point(i);
                    indices[3] = NDArrayIndex.all();
                } else {
                    indices[2] = NDArrayIndex.all();
                    indices[3] = NDArrayIndex.point(i);
                }

                features.put(indices, vector);
            }

            return features;
        }
    }
769757
@Override
    protected List<Term> segSentence(char[] sentence)
    {
        if (sentence.length == 0) return Collections.emptyList();
        List<Term> termList = roughSegSentence(sentence);
        if (!(config.ner || config.useCustomDictionary || config.speechTagging))
            return termList;
        List<Vertex> vertexList = toVertexList(termList, true);
        if (config.speechTagging)
        {
            Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
            int i = 0;
            for (Term term : termList)
            {
                if (term.nature != null) term.nature = vertexList.get(i + 1).guessNature();
                ++i;
            }
        }
        if (config.useCustomDictionary)
        {
            combineByCustomDictionary(vertexList);
            termList = convert(vertexList, config.offset);
        }
        return termList;
    }
508684
function compareTaggedSentences(sentenceInOldFormat, sentenceInNewFormat) {
  var equal = true;
  sentenceInOldFormat.forEach(function(wordPlusTag, index) {
    equal = equal &&
      (wordPlusTag[1] === sentenceInNewFormat.taggedWords[index].tag);
    DEBUG && console.log(wordPlusTag[1] + ' ' + sentenceInNewFormat.taggedWords[index].tag);
  });
  return equal;
}
767683
@Override
  public RuleMatch[] match(AnalyzedSentence sentence) throws IOException {
    List<RuleMatch> ruleMatches = new ArrayList<>();

    // Let's get all the tokens (i.e. words) of this sentence, but not the spaces:
    AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
    
    // No let's iterate over those - note that the first token will
    // be a special token that indicates the start of a sentence:
    for (AnalyzedTokenReadings token : tokens) {
      
      System.out.println("Token: " + token.getToken());  // the original word from the input text
      
      // A word can have more than one reading, e.g. 'dance' can be a verb or a noun,
      // so we iterate over the readings:
      for (AnalyzedToken analyzedToken : token.getReadings()) {
        System.out.println("  Lemma: " + analyzedToken.getLemma());
        System.out.println("  POS: " + analyzedToken.getPOSTag());
      }
      
      // You can add your own logic here to find errors. Here, we just consider
      // the word "demo" an error and create a rule match that LanguageTool will
      // then show to the user:
      if (token.getToken().equals("demo")) {
        RuleMatch ruleMatch = new RuleMatch(this, sentence, token.getStartPos(), token.getEndPos(), "The demo rule thinks this looks wrong");
        ruleMatch.setSuggestedReplacement("blablah");  // the user will see this as a suggested correction
        ruleMatches.add(ruleMatch);
      }
    }

    return toRuleMatchArray(ruleMatches);
  }
1150591
def sentence_iterator(self, file_path: str) -> Iterator[OntonotesSentence]:
        
        for document in self.dataset_document_iterator(file_path):
            for sentence in document:
                yield sentence
3584
private Sentence constructSentence(List<Token> tokens) throws IOException {
        Sentence sentence;
        try {
            sentence = new SimpleSentence(tokens, strict);
        } catch (IllegalArgumentException e) {
            throw new IOException(e.getMessage());
        }
        return sentence;
    }
394494
def sentences(self):
        """"""
        prevp = 0
        prevs = 0
        sentence = [];
        sentence_id = ""
        for word, id, pos, lemma in iter(self):
            try:
                doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if ((p != prevp) or (s != prevs)) and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.' + ptype + '.' + str(p) + '.s.' + str(s)
                prevp = p
            except IndexError:
                doc_id, s, w = re.findall('([\w\d-]+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if s != prevs and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.s.' + str(s)
            sentence.append( (word,id,pos,lemma) )
            prevs = s
        if sentence:
            yield sentence_id, sentence
1168418
public static String sentences(int sentenceCount, boolean supplemental) {
        List<String> sentenceList = new ArrayList<String>();
        for (int i = 0; i < sentenceCount; i++) {
            sentenceList.add(sentence(3, supplemental, 6));
        }

        String sentenceString = StringUtils.join(sentenceList, " ");
        return sentenceString;

    }
925596
def _add_sentence(self, sentence, ignore_traces=True):
        
        self.sentences.append(self._node_id)
        # add edge from document root to sentence root
        self.add_edge(self.root, self._node_id, edge_type=dg.EdgeTypes.dominance_relation)
        self._parse_sentencetree(sentence, ignore_traces=ignore_traces)
        self._node_id += 1
65088
public static Window windowForWordInPosition(int windowSize, int wordPos, List<String> sentence) {
        List<String> window = new ArrayList<>();
        List<String> onlyTokens = new ArrayList<>();
        int contextSize = (int) Math.floor((windowSize - 1) / 2);

        for (int i = wordPos - contextSize; i <= wordPos + contextSize; i++) {
            if (i < 0)
                window.add("<s>");
            else if (i >= sentence.size())
                window.add("</s>");
            else {
                onlyTokens.add(sentence.get(i));
                window.add(sentence.get(i));

            }
        }

        String wholeSentence = StringUtils.join(sentence);
        String window2 = StringUtils.join(onlyTokens);
        int begin = wholeSentence.indexOf(window2);
        int end = begin + window2.length();
        return new Window(window, begin, end);

    }
584405
private List<List<Term>> seg2sentence(String text) {
        List<String> sentenceList = toSentenceList(text);
        List<List<Term>> resultList = new ArrayList<List<Term>>(sentenceList.size());
        for (String sentence : sentenceList) {
            resultList.add(analysis.parseStr(sentence).recognition(sr).getTerms());
        }
        return resultList;
    }
358725
public void classifyAndWriteAnswersKBest(ObjectBank<List<IN>> documents, int k, PrintWriter printWriter,
                                           DocumentReaderAndWriter<IN> readerAndWriter) throws IOException {
    Timing timer = new Timing();
    int numWords = 0;
    int numSentences = 0;

    for (List<IN> doc : documents) {
      Counter<List<IN>> kBest = classifyKBest(doc, AnswerAnnotation.class, k);
      numWords += doc.size();
      List<List<IN>> sorted = Counters.toSortedList(kBest);
      int n = 1;
      for (List<IN> l : sorted) {
        System.out.println("<sentence id=" + numSentences + " k=" + n + " logProb=" + kBest.getCount(l) + " prob="
            + Math.exp(kBest.getCount(l)) + '>');
        writeAnswers(l, printWriter, readerAndWriter);
        System.out.println("</sentence>");
        n++;
      }
      numSentences++;
    }

    long millis = timer.stop();
    double wordspersec = numWords / (((double) millis) / 1000);
    NumberFormat nf = new DecimalFormat("0.00"); // easier way!
    System.err.println(this.getClass().getName() + " tagged " + numWords + " words in " + numSentences
        + " documents at " + nf.format(wordspersec) + " words per second.");
  }
304903
def next(self, type=None):
        
        i = self.index + 1
        s = self.sentence
        while i < len(s):
            if type in (s[i].type, None):
                return s[i]
            i += 1
603150
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // Prefetch the first windowSize words.  As soon as a word enters the
        // nextWords buffer increase its occurrence count.
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i) 
            nextWords.offer(documentTokens.next());        
        
        while (!nextWords.isEmpty()) {
            
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    
                
            // Don't bother calculating the semantics for empty tokens
            // (i.e. words that were filtered out)
            if (!focusWord.equals(IteratorFactory.EMPTY_TOKEN)) {
                SemanticVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                            
                    updateSemantics(focusMeaning, word, iv);
                }
                
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    updateSemantics(focusMeaning, word, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);

            // Increment the frequency count for the word now that it has been
            // seen and processed.
            Integer count = wordToOccurrences.get(focusWord);
            wordToOccurrences.put(focusWord, (count == null) ? 1 : count + 1);

            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }