Adds co-occurrence counts between the list of previous words in {@code
words} and the focus word represented by {@code focusIndex} which start
at {@code distance} tokens away from the focus word.  All Counts will be
added into {@code matrixEntryToCount}.
353187
private void addTokens(Queue<String> words,
                           int focusIndex,
                           int distance,
                           Map<Pair<Integer>, Double> matrixEntryToCount) {
        for (String word : words) {
            // skip adding co-occurence values for words that are not
            // accepted by the filter
            if (!word.equals(IteratorFactory.EMPTY_TOKEN)) {
                int index = termToIndex.getDimension(word);
                if (index >= 0) {
                    // Get the current number of times that the focus word has
                    // co-occurred with this word before after it.  Weight the
                    // word appropriately baed on distance
                    Pair<Integer> p = new Pair<Integer>(index, focusIndex);
                    double value = weighting.weight(distance, windowSize);
                    Double curCount = matrixEntryToCount.get(p);
                    matrixEntryToCount.put(p, (curCount == null)
                                           ? value : value + curCount);
                }
            }
            distance++;
        }
    }
923105
public void addWords(Collection<String> words){
		if(false == (words instanceof Set)){
			words = new HashSet<>(words);
		}
		for (String word : words) {
			addWord(word);
		}
	}
235192
public boolean createDictionaryEntry(String key, long count, SuggestionStage staging) {
    if (count <= 0) {
      if (this.countThreshold > 0) {
        return false; // no point doing anything if count is zero, as it can't change anything
      }
      count = 0;
    }
    long countPrevious;

    // look first in below threshold words, update count, and allow promotion to correct spelling word if count reaches threshold
    // threshold must be >1 for there to be the possibility of low threshold words
    if (countThreshold > 1 && belowThresholdWords.containsKey(key)) {
      countPrevious = belowThresholdWords.get(key);
      // calculate new count for below threshold word
      count = (Long.MAX_VALUE - countPrevious > count) ? countPrevious + count : Long.MAX_VALUE;
      // has reached threshold - remove from below threshold collection (it will be added to correct words below)
      if (count >= countThreshold) {
        belowThresholdWords.remove(key);
      } else {
        belowThresholdWords.put(key, count); // = count;
        return false;
      }
    } else if (words.containsKey(key)) {
      countPrevious = words.get(key);
      // just update count if it's an already added above threshold word
      count = (Long.MAX_VALUE - countPrevious > count) ? countPrevious + count : Long.MAX_VALUE;
      words.put(key, count);
      return false;
    } else if (count < countThreshold) {
      // new or existing below threshold word
      belowThresholdWords.put(key, count);
      return false;
    }

    // what we have at this point is a new, above threshold word
    words.put(key, count);
    if (key.equals("can't")) {
      System.out.println("Added to words..!");
    }

    //edits/suggestions are created only once, no matter how often word occurs
    //edits/suggestions are created only as soon as the word occurs in the corpus,
    //even if the same term existed before in the dictionary as an edit from another word
    if (key.length() > maxLength) {
      maxLength = key.length();
    }

    //create deletes
    HashSet<String> edits = editsPrefix(key);

    // if not staging suggestions, put directly into main data structure
    if (staging != null) {
      edits.forEach(delete -> staging.add(getStringHash(delete), key));
    } else {
      if (deletes == null) {
        this.deletes = new HashMap<>(initialCapacity); //initialisierung
      }

      edits.forEach(delete -> {
        int deleteHash = getStringHash(delete);
        String[] suggestions;
        if (deletes.containsKey(deleteHash)) {
          suggestions = deletes.get(deleteHash);
          String[] newSuggestions = Arrays.copyOf(suggestions, suggestions.length + 1);
          deletes.put(deleteHash, newSuggestions);
          suggestions = newSuggestions;
        } else {
          suggestions = new String[1];
          deletes.put(deleteHash, suggestions);
        }
        suggestions[suggestions.length - 1] = key;
      });
    }
    return true;
  }
431506
private void addMacroCodewords() {

        // if the structured append series size is 1, this isn't
        // actually part of a structured append series
        if (structuredAppendTotal == 1) {
            return;
        }

        // add the Macro marker codeword
        codeWords[codeWordCount++] = 928;

        // add the segment index, padded with leading zeros to five digits
        // use numeric compaction, but no latch
        int segmentIndex = structuredAppendPosition - 1;
        int[] data = new int[5];
        for (int x = data.length - 1; x >= 0; x--) {
            data[x] = '0' + (segmentIndex % 10);
            segmentIndex /= 10;
        }
        processNumbers(data, 0, data.length, true);

        // add the file ID (base 900, which is easy since we limit
        // file ID values to the range 0 to 899)
        codeWords[codeWordCount++] = structuredAppendFileId;

        // NOTE: we could add the optional segment count field here, but
        // it doesn't appear to be necessary... if we do eventually decide
        // to add it, it will probably be [923, 001, count1, count2]

        // add the terminator to the last symbol of the series
        boolean last = (structuredAppendPosition == structuredAppendTotal);
        if (last) {
            codeWords[codeWordCount++] = 922;
        }
    }
23146
private void addEvents(String sentence) {
    String[] parts = sentence.split(" ");
    String outcome = NameFinderME.OTHER;
    List toks = new ArrayList();
    List outcomes = new ArrayList();
    for (int pi = 0, pl = parts.length; pi < pl; pi++) {
      if (parts[pi].equals("<START>")) {
        outcome = NameFinderME.START;
      }
      else if (parts[pi].equals("<END>")) {
        outcome = NameFinderME.OTHER;
      }
      else { //regular token
        toks.add(parts[pi]);
        outcomes.add(outcome);
        if (outcome.equals(NameFinderME.START)) {
          outcome = NameFinderME.CONTINUE;
        }
      }
    }
    events = new Event[toks.size()];
    for (int ti = 0, tl = toks.size(); ti < tl; ti++) {
      events[ti] = new Event((String) outcomes.get(ti), cg.getContext(ti, toks, outcomes, prevTags));
    }
    for (int ti=0,tl=toks.size();ti<tl;ti++) {
      prevTags.put(toks.get(ti),outcomes.get(ti));
    }
  }
901060
protected void addTagging(boolean seen, IntTaggedWord itw, double count) {
    if (seen) {
      seenCounter.incrementCount(itw, count);
      if (itw.tag() == nullTag) {
        words.add(itw);
      } else if (itw.word() == nullWord) {
        tags.add(itw);
      } else {
        // rules.add(itw);
      }
    } else {
      uwModel.addTagging(seen, itw, count);
      // if (itw.tag() == nullTag) {
      // sigs.add(itw);
      // }
    }
  }
880336
@Override
    public INDArray getWordVectors(Collection<String> labels) {
        List<INDArray> words = new ArrayList<>();
        for (String label : labels) {
            if (hasWord(label) || getUNK() != null)
                words.add(getWordVectorMatrix(label));
        }

        return Nd4j.vstack(words);
    }
1021034
static <T> List<T> toList(final Class<T> targetClass, final Cursor cursor, final int columnIndex, int offset, int count) {
        if (columnIndex < 0 || columnIndex >= cursor.getColumnCount()) {
            throw new IllegalArgumentException("Invalid column index: " + columnIndex);
        }

        if (offset > 0) {
            cursor.moveToPosition(offset - 1);
        }

        final List<T> resultList = new ArrayList<>();

        if (N.isEntity(targetClass)) {
            final Method propSetMethod = ClassUtil.getPropSetMethod(targetClass, cursor.getColumnName(columnIndex));
            final Type<T> selectColumnType = Type.valueOf(propSetMethod.getParameterTypes()[0]);

            while (count-- > 0 && cursor.moveToNext()) {
                T entity = N.newEntity(targetClass);
                ClassUtil.setPropValue(entity, propSetMethod, selectColumnType.get(cursor, columnIndex));
                resultList.add(entity);
            }

        } else {
            final Type<T> selectColumnType = Type.valueOf(targetClass);

            while (count-- > 0 && cursor.moveToNext()) {
                resultList.add(selectColumnType.get(cursor, columnIndex));
            }
        }

        return resultList;
    }
603150
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // Prefetch the first windowSize words.  As soon as a word enters the
        // nextWords buffer increase its occurrence count.
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i) 
            nextWords.offer(documentTokens.next());        
        
        while (!nextWords.isEmpty()) {
            
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    
                
            // Don't bother calculating the semantics for empty tokens
            // (i.e. words that were filtered out)
            if (!focusWord.equals(IteratorFactory.EMPTY_TOKEN)) {
                SemanticVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                            
                    updateSemantics(focusMeaning, word, iv);
                }
                
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    updateSemantics(focusMeaning, word, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);

            // Increment the frequency count for the word now that it has been
            // seen and processed.
            Integer count = wordToOccurrences.get(focusWord);
            wordToOccurrences.put(focusWord, (count == null) ? 1 : count + 1);

            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
1102925
public Integer addTokenAnnotation(String tokenString, String fileId, Integer sentId, Integer tokId, Integer positionCounter, JCas jcas){
		Token token = new Token(jcas);
		if (!((sentId == newTokSentNumber) && (tokId == newTokSentNumber))){
			if(USE_SPACES) // in chinese, there are no spaces, so the +1 correction is unnecessary
				positionCounter = positionCounter + 1;
		}
		token.setBegin(positionCounter);
		positionCounter = positionCounter + tokenString.length();
		token.setEnd(positionCounter);
		token.setTokenId(tokId);
		token.setSentId(sentId);
		token.setFilename(fileId);
		token.addToIndexes();
		
		String id = fileId+"_"+sentId+"_"+tokId;
		hmToken.put(id, token);
		
		return positionCounter;
  }
1370152
public boolean add( T word )
    {
        Entry<T> o = new Entry<T>(word, tail.prev, tail);
        tail.prev.next = o;
        tail.prev = o;
        
        //set the size and set the index
        size++;
        index.put(word.getValue(), word);
        
        return true;
    }
923930
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        //用词频来标注词的权重
        taggingWeightWithWordFrequency(words1, words2);
        //构造权重快速搜索容器
        Map<String, Float> weights1 = toFastSearchMap(words1);
        Map<String, Float> weights2 = toFastSearchMap(words2);
        //所有的不重复词
        Set<Word> words = new HashSet<>();
        words.addAll(words1);
        words.addAll(words2);
        //向量的维度为words的大小，每一个维度的权重是词频
        //a.b
        AtomicFloat ab = new AtomicFloat();
        //|a|的平方
        AtomicFloat aa = new AtomicFloat();
        //|b|的平方
        AtomicFloat bb = new AtomicFloat();
        //计算
        words
            .parallelStream()
            .forEach(word -> {
                Float x1 = weights1.get(word.getText());
                Float x2 = weights2.get(word.getText());
                if (x1 != null && x2 != null) {
                    //x1x2
                    float oneOfTheDimension = x1 * x2;
                    //+
                    ab.addAndGet(oneOfTheDimension);
                }
                if (x1 != null) {
                    //(x1)^2
                    float oneOfTheDimension = x1 * x1;
                    //+
                    aa.addAndGet(oneOfTheDimension);
                }
                if (x2 != null) {
                    //(x2)^2
                    float oneOfTheDimension = x2 * x2;
                    //+
                    bb.addAndGet(oneOfTheDimension);
                }
            });
        //|a|
        double aaa = Math.sqrt(aa.doubleValue());
        //|b|
        double bbb = Math.sqrt(bb.doubleValue());
        //使用BigDecimal保证精确计算浮点数
        //|a|*|b|
        //double aabb = aaa * bbb;
        BigDecimal aabb = BigDecimal.valueOf(aaa).multiply(BigDecimal.valueOf(bbb));
        //similarity=a.b/|a|*|b|
        //double cos = ab.get() / aabb.doubleValue();
        double cos = BigDecimal.valueOf(ab.get()).divide(aabb, 9, BigDecimal.ROUND_HALF_UP).doubleValue();
        return cos;
    }
989900
private PriorityQueue<Object[]> createQueue(Map<String, Int> words) throws IOException
   {
      // have collected all words in doc and their freqs
      int numDocs = ir.numDocs();
      FreqQ res = new FreqQ(words.size()); // will order words by score

      Iterator<String> it = words.keySet().iterator();
      while (it.hasNext())
      { // for every word
         String word = it.next();

         int tf = words.get(word).x; // term freq in the source doc
         if (minTermFreq > 0 && tf < minTermFreq)
         {
            continue; // filter out words that don't occur enough times in the source
         }

         // go through all the fields and find the largest document frequency
         String topField = fieldNames[0];
         int docFreq = 0;
         for (int i = 0; i < fieldNames.length; i++)
         {
            int freq = ir.docFreq(new Term(fieldNames[i], word));
            topField = (freq > docFreq) ? fieldNames[i] : topField; //NOSONAR
            docFreq = (freq > docFreq) ? freq : docFreq;
         }

         if (minDocFreq > 0 && docFreq < minDocFreq)
         {
            continue; // filter out words that don't occur in enough docs
         }

         if (docFreq == 0)
         {
            continue; // index update problem?
         }

         float idf = similarity.idf(docFreq, numDocs);
         float score = tf * idf;

         // only really need 1st 3 entries, other ones are for troubleshooting
         res.insertWithOverflow(new Object[]{word, // the word
            topField, // the top field
            new Float(score), // overall score
            new Float(idf), // idf
            new Integer(docFreq), // freq in all docs
            new Integer(tf)});
      }
      return res;
   }
3451
protected List<CRFDatum> extractDatumSequence(int[][][] allData, int beginPosition, int endPosition,
      List<IN> labeledWordInfos) {
    List<CRFDatum> result = new ArrayList<CRFDatum>();
    int beginContext = beginPosition - windowSize + 1;
    if (beginContext < 0) {
      beginContext = 0;
    }
    // for the beginning context, add some dummy datums with no features!
    // TODO: is there any better way to do this?
    for (int position = beginContext; position < beginPosition; position++) {
      List<Collection<String>> cliqueFeatures = new ArrayList<Collection<String>>();
      for (int i = 0; i < windowSize; i++) {
        // create a feature list
        cliqueFeatures.add(Collections.<String>emptyList());
      }
      CRFDatum<Collection<String>, String> datum = new CRFDatum<Collection<String>, String>(cliqueFeatures,
              labeledWordInfos.get(position).get(AnswerAnnotation.class));
      result.add(datum);
    }
    // now add the real datums
    for (int position = beginPosition; position <= endPosition; position++) {
      List<Collection<String>> cliqueFeatures = new ArrayList<Collection<String>>();
      for (int i = 0; i < windowSize; i++) {
        // create a feature list
        Collection<String> features = new ArrayList<String>();
        for (int j = 0; j < allData[position][i].length; j++) {
          features.add(featureIndex.get(allData[position][i][j]));
        }
        cliqueFeatures.add(features);
      }
      CRFDatum<Collection<String>,String> datum = new CRFDatum<Collection<String>,String>(cliqueFeatures,
              labeledWordInfos.get(position).get(AnswerAnnotation.class));
      result.add(datum);
    }
    return result;
  }
249505
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2){
        double score = super.scoreImpl(words1, words2);

        score += (scalingFactor * commonPrefixLength() * (1.0 - score));

        return score;
    }
479960
private void addStopwords(String dir) {
		if (dir == null) {
			logger.info("no stopwords dir");
			return;
		}
		logger.info("stopwords: {}", dir);
		filter = new HashSet<String>();
		File file = new File(dir);
		try (FileInputStream fis = new FileInputStream(file)) {
			InputStreamReader reader = new InputStreamReader(fis, "UTF-8");
			BufferedReader br = new BufferedReader(reader);
			String word = br.readLine();
			while (word != null) {
				filter.add(word);
				word = br.readLine();
			}
		} catch (FileNotFoundException e) {
			logger.info("No stopword file found");
		} catch (IOException e) {
			logger.info("stopword file io exception");
		}
	}
811877
private static void contextWord(String[] words, int index, int distance, String word, Map<String, List<ContextWord>> map){
        String _word = null;
        if(index > -1 && index < words.length){
            _word = words[index];                    
        }
        if(_word != null && Utils.isChineseCharAndLengthAtLeastTwo(_word)){
            addToMap(map, word, _word, distance);
        }
    }