For each word, adjust it's score by usage

 v = s * (1 - a) + u * a
 Where v is the new value
 a is @alpha
 s is the bigram score (0..1)
 u is the usage score (0..1)
515161
def apply_usage_weights(word_hash)
      max_usage = @word_list.values.max.to_f
      max_usage = 1 if max_usage == 0

      weighted_array = word_hash.map do |word, bigram_score|
        usage_score = @word_list[word].to_f / max_usage
        [word, (bigram_score * (1 - @alpha)) + (usage_score * @alpha)]
      end

      Hash[weighted_array]
    end
961342
def adjustHspsForPlotting(self, titleAlignments):
        
        # If we're using bit scores, there's nothing to do.
        if self.scoreClass is HigherIsBetterScore:
            return

        # Convert all e-values to high positive values, and keep track of the
        # maximum converted value.
        maxConvertedEValue = None
        zeroHsps = []

        # Note: don't call self.hsps() here because that will read them
        # from disk again, which is not what's wanted.
        for hsp in titleAlignments.hsps():
            if hsp.score.score == 0.0:
                zeroHsps.append(hsp)
            else:
                convertedEValue = -1.0 * log10(hsp.score.score)
                hsp.score.score = convertedEValue
                if (maxConvertedEValue is None or
                        convertedEValue > maxConvertedEValue):
                    maxConvertedEValue = convertedEValue

        if zeroHsps:
            # Save values so that we can use them in self.adjustPlot
            self._maxConvertedEValue = maxConvertedEValue
            self._zeroEValueFound = True

            # Adjust all zero e-value HSPs to have numerically high values.
            if self.randomizeZeroEValues:
                for hsp in zeroHsps:
                    hsp.score.score = (maxConvertedEValue + 2 + uniform(
                        0, ZERO_EVALUE_UPPER_RANDOM_INCREMENT))
            else:
                for count, hsp in enumerate(zeroHsps, start=1):
                    hsp.score.score = maxConvertedEValue + count
        else:
            self._zeroEValueFound = False
1128644
public double iterateSample(T w1, T w2, double score) {
        INDArray w1Vector = syn0.slice(w1.getIndex());
        INDArray w2Vector = syn0.slice(w2.getIndex());
        //prediction: input + bias
        if (w1.getIndex() < 0 || w1.getIndex() >= syn0.rows())
            throw new IllegalArgumentException("Illegal index for word " + w1.getLabel());
        if (w2.getIndex() < 0 || w2.getIndex() >= syn0.rows())
            throw new IllegalArgumentException("Illegal index for word " + w2.getLabel());


        //w1 * w2 + bias
        double prediction = Nd4j.getBlasWrapper().dot(w1Vector, w2Vector);
        prediction += bias.getDouble(w1.getIndex()) + bias.getDouble(w2.getIndex());

        double weight = Math.pow(Math.min(1.0, (score / maxCount)), xMax);

        double fDiff = score > xMax ? prediction : weight * (prediction - Math.log(score));
        if (Double.isNaN(fDiff))
            fDiff = Nd4j.EPS_THRESHOLD;
        //amount of change
        double gradient = fDiff;

        //note the update step here: the gradient is
        //the gradient of the OPPOSITE word
        //for adagrad we will use the index of the word passed in
        //for the gradient calculation we will use the context vector
        update(w1, w1Vector, w2Vector, gradient);
        update(w2, w2Vector, w1Vector, gradient);
        return fDiff;



    }
559532
def best_match(given_word)
      words = (@word_list.is_a? Array) ? @word_list : @word_list.keys

      word_bigrams = bigramate(given_word)
      word_hash = words.map do |key|
        [key, bigram_compare(word_bigrams, bigramate(key))]
      end
      word_hash = Hash[word_hash]

      # Weight by word usage, if logical
      word_hash = apply_usage_weights(word_hash) if @word_list.is_a? Hash

      word_hash.max_by { |_key, value| value }.first
    end
1181625
def get_total_alignment_score(bam):
    ''''''
    sam_reader = pysam.Samfile(bam, "rb")
    total = 0
    for sam in sam_reader.fetch(until_eof=True):
        try:
            total += sam.opt('AS')
        except:
            pass
    return total
429616
private Counter<L> scoresOfRVFDatum(RVFDatum<L, F> example) {
    Counter<L> scores = new ClassicCounter<L>();
    for (L l : labels()) {
      scores.setCount(l, scoreOfRVFDatum(example, l));
    }
    //System.out.println("Scores are: " + scores + "   (gold: " + example.label() + ")");
    return scores;
  }
421400
protected void updateDensities(WritableDoubleDataStore rbod_score, DoubleDBIDList referenceDists) {
    DoubleDBIDListIter it = referenceDists.iter();
    for(int l = 0; l < referenceDists.size(); l++) {
      double density = computeDensity(referenceDists, it, l);
      // computeDensity modified the iterator, reset:
      it.seek(l);
      // NaN indicates the first run.
      if(!(density > rbod_score.doubleValue(it))) {
        rbod_score.putDouble(it, density);
      }
    }
  }
485740
function(dataset) {

		// calculate counts for equation (3):
		var mapWordToTotalCount = {};
		var totalNumberOfWordsInDataset = 0;
		for (var i in dataset) {
			var datum = dataset[i];
			var totalPerDatum = 0;
			
			// for each input sentence, count the total number of words in it:
			for (var word in datum) {
				mapWordToTotalCount[word] |= 0;
				mapWordToTotalCount[word] += datum[word];
				totalPerDatum += datum[word];
			}
			datum["_total"] = totalPerDatum;
			totalNumberOfWordsInDataset += totalPerDatum;
		}
		mapWordToTotalCount["_total"] = totalNumberOfWordsInDataset;
		
		this.dataset = dataset;
		this.mapWordToTotalCount = mapWordToTotalCount;
		
		// calculate smoothing factor for equation (3):
		var mapWordToSmoothingFactor = {};
		for (var word in mapWordToTotalCount) {
			mapWordToSmoothingFactor[word] = 
				(1-this.smoothingCoefficient) * this.mapWordToTotalCount[word] / this.mapWordToTotalCount["_total"];
		}
		this.mapWordToSmoothingFactor = mapWordToSmoothingFactor;
		
		this.globalSmoothingFactor = (1/totalNumberOfWordsInDataset)  // a global smoother, for totally unseen words.
	}
579871
def get_score(self, terms):
        
        assert isinstance(terms, list) or isinstance(terms, tuple)
        score_li = np.asarray([self._get_score(t) for t in terms])
        
        s_pos = np.sum(score_li[score_li > 0])
        s_neg = -np.sum(score_li[score_li < 0])
        
        s_pol = (s_pos-s_neg) * 1.0 / ((s_pos+s_neg)+self.EPSILON)
        s_sub = (s_pos+s_neg) * 1.0 / (len(score_li)+self.EPSILON)
        
        return {self.TAG_POS: s_pos,
                self.TAG_NEG: s_neg,
                self.TAG_POL: s_pol,
                self.TAG_SUB: s_sub}
477056
@Override
    @Deprecated
    public void iterateSample(T w1, T w2, AtomicLong nextRandom, double alpha) {
        if (w2 == null || w2.getIndex() < 0 || w1.getIndex() == w2.getIndex() || w1.getLabel().equals("STOP")
                        || w2.getLabel().equals("STOP") || w1.getLabel().equals("UNK") || w2.getLabel().equals("UNK"))
            return;
        //current word vector
        INDArray l1 = this.syn0.slice(w2.getIndex());


        //error for current word and context
        INDArray neu1e = Nd4j.create(vectorLength);


        for (int i = 0; i < w1.getCodeLength(); i++) {
            int code = w1.getCodes().get(i);
            int point = w1.getPoints().get(i);
            if (point >= syn0.rows() || point < 0)
                throw new IllegalStateException("Illegal point " + point);
            //other word vector

            INDArray syn1 = this.syn1.slice(point);


            double dot = Nd4j.getBlasWrapper().dot(l1, syn1);

            if (dot < -MAX_EXP || dot >= MAX_EXP)
                continue;


            int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0));
            if (idx >= expTable.length)
                continue;

            //score
            double f = expTable[idx];
            //gradient
            double g = useAdaGrad ? w1.getGradient(i, (1 - code - f), lr.get()) : (1 - code - f) * alpha;

            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, syn1, neu1e);
            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, l1, syn1);

        }


        int target = w1.getIndex();
        int label;
        //negative sampling
        if (negative > 0)
            for (int d = 0; d < negative + 1; d++) {
                if (d == 0)
                    label = 1;
                else {
                    nextRandom.set(nextRandom.get() * 25214903917L + 11);

                    // FIXME: int cast
                    int idx = (int) Math.abs((int) (nextRandom.get() >> 16) % table.length());

                    target = table.getInt(idx);
                    if (target <= 0)
                        target = (int) nextRandom.get() % (vocab.numWords() - 1) + 1;

                    if (target == w1.getIndex())
                        continue;
                    label = 0;
                }


                if (target >= syn1Neg.rows() || target < 0)
                    continue;

                double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg.slice(target));
                double g;
                if (f > MAX_EXP)
                    g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha;
                else if (f < -MAX_EXP)
                    g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha);
                else
                    g = useAdaGrad ? w1
                                    .getGradient(target,
                                                    label - expTable[(int) ((f + MAX_EXP)
                                                                    * (expTable.length / MAX_EXP / 2))],
                                                    alpha)
                                    : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))])
                                                    * alpha;
                if (syn0.data().dataType() == DataType.DOUBLE)
                    Nd4j.getBlasWrapper().axpy(g, syn1Neg.slice(target), neu1e);
                else
                    Nd4j.getBlasWrapper().axpy((float) g, syn1Neg.slice(target), neu1e);

                if (syn0.data().dataType() == DataType.DOUBLE)
                    Nd4j.getBlasWrapper().axpy(g, l1, syn1Neg.slice(target));
                else
                    Nd4j.getBlasWrapper().axpy((float) g, l1, syn1Neg.slice(target));
            }

        if (syn0.data().dataType() == DataType.DOUBLE)
            Nd4j.getBlasWrapper().axpy(1.0, neu1e, l1);

        else
            Nd4j.getBlasWrapper().axpy(1.0f, neu1e, l1);

    }
62346
def markov_update_count!(grams, word, gram_size)
      word.chars.each_cons(gram_size + 1) do |gram|
        l = gram[0..gram_size - 1].join
        r = gram[gram_size]

        unless l.empty? || r.empty? || line_ending?(r)
          grams[l] = {} if grams[l].nil?
          grams[l][r] = grams[l][r].nil? ? 1 : grams[l][r] += 1
        end
      end
    end
1087066
def lt(self, v, limit=None, offset=None):
        
        if limit is not None and offset is None:
            offset = 0
        return self.zrangebyscore(self._min_score, "(%f" % v,
                start=offset, num=limit)
965987
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        if(words1.isEmpty() && words2.isEmpty()){
            return 1.0;
        }
        //HashSet的contains性能要大于ArrayList的contains
        Set<Word> words2Set = new HashSet<>();
        words2Set.addAll(words2);
        //求交集
        Set<String> intersectionSet = new ConcurrentSkipListSet<>();
        words1.parallelStream().forEach(word -> {
            if (words2Set.contains(word)) {
                intersectionSet.add(word.getText());
            }
        });
        //交集的大小
        int intersectionSize = intersectionSet.size();
        //求并集
        Set<String> unionSet = new HashSet<>();
        words1.forEach(word -> unionSet.add(word.getText()));
        words2.forEach(word -> unionSet.add(word.getText()));
        //并集的大小
        int unionSize = unionSet.size();
        //相似度分值
        double score = intersectionSize/(double)unionSize;
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("交集的大小：" + intersectionSize);
            LOGGER.debug("并集的大小：" + unionSize);
            LOGGER.debug("相似度分值=" + intersectionSize + "/(double)" + unionSize + "=" + score);
        }
        return score;
    }
1153559
def standardize(score):
    """"""
    score = asarray_ndim(score, 1)
    return (score - np.nanmean(score)) / np.nanstd(score)
440594
def scorerange(self, start, end, start_inclusive=True, end_inclusive=True):
        
        if not self:
            return []

        left = bisect_left(self._scores, (start,))
        right = bisect_right(self._scores, (end,))

        if end_inclusive:
            # end is inclusive
            while right < len(self) and self._scores[right][0] == end:
                right += 1

        if not start_inclusive:
            while left < right and self._scores[left][0] == start:
                left += 1
        return self._scores[left:right]
101929
public double sim(List<String> sentence, int index)
    {
        double score = 0;
        for (String word : sentence)
        {
            if (!f[index].containsKey(word)) continue;
            int d = docs.get(index).size();
            Integer tf = f[index].get(word);
            score += (idf.get(word) * tf * (k1 + 1)
                    / (tf + k1 * (1 - b + b * d
                                                / avgdl)));
        }

        return score;
    }
833370
function(sentences, numberOfTopics, numberOfTermsPerTopic, languages, alphaValue, betaValue, randomSeed) {
    // The result will consist of topics and their included terms [[{"term":"word1", "probability":0.065}, {"term":"word2", "probability":0.047}, ... ], [{"term":"word1", "probability":0.085}, {"term":"word2", "probability":0.024}, ... ]].
    var result = [];
    // Index-encoded array of sentences, with each row containing the indices of the words in the vocabulary.
    var documents = new Array();
    // Hash of vocabulary words and the count of how many times each word has been seen.
    var f = {};
    // Vocabulary of unique words (porter stemmed).
    var vocab=new Array();
    // Vocabulary of unique words in their original form.
    var vocabOrig = {};
    // Array of stop words
    languages = languages || Array('en');

    if (sentences && sentences.length > 0) {
      var stopwords = new Array();

      languages.forEach(function(value) {
          var stopwordsLang = require('./stopwords_' + value + ".js");
          stopwords = stopwords.concat(stopwordsLang.stop_words);
      });

      for(var i=0;i<sentences.length;i++) {
          if (sentences[i]=="") continue;
          documents[i] = new Array();

          var words = sentences[i] ? sentences[i].split(/[\s,\"]+/) : null;

          if(!words) continue;
          for(var wc=0;wc<words.length;wc++) {
              var w=words[wc].toLowerCase().replace(/[^a-z\'A-Z0-9\u00C0-\u00ff ]+/g, '');
              var wStemmed = stem(w);
              if (w=="" || !wStemmed || w.length==1 || stopwords.indexOf(w.replace("'", "")) > -1 || stopwords.indexOf(wStemmed) > -1 || w.indexOf("http")==0) continue;
              if (f[wStemmed]) { 
                  f[wStemmed]=f[wStemmed]+1;
              } 
              else if(wStemmed) { 
                  f[wStemmed]=1; 
                  vocab.push(wStemmed);
                  vocabOrig[wStemmed] = w;
              };
              
              documents[i].push(vocab.indexOf(wStemmed));
          }
      }

      var V = vocab.length;
      var M = documents.length;
      var K = parseInt(numberOfTopics);
      var alpha = alphaValue || 0.1;  // per-document distributions over topics
      var beta = betaValue || .01;  // per-topic distributions over words
      documents = documents.filter((doc) => { return doc.length }); // filter empty documents

      lda.configure(documents,V,10000, 2000, 100, 10, randomSeed);
      lda.gibbs(K, alpha, beta);

      var theta = lda.getTheta();
      var phi = lda.getPhi();

      var text = '';

      //topics
      var topTerms=numberOfTermsPerTopic;
      for (var k = 0; k < phi.length; k++) {
          var things = new Array();
          for (var w = 0; w < phi[k].length; w++) {
               things.push(""+phi[k][w].toPrecision(2)+"_"+vocab[w] + "_" + vocabOrig[vocab[w]]);
          }
          things.sort().reverse();
          //console.log(things);
          if(topTerms>vocab.length) topTerms=vocab.length;

          //console.log('Topic ' + (k + 1));
          var row = [];
          
          for (var t = 0; t < topTerms; t++) {
              var topicTerm=things[t].split("_")[2];
              var prob=parseInt(things[t].split("_")[0]*100);
              if (prob<2) continue;
              
              //console.log('Top Term: ' + topicTerm + ' (' + prob + '%)');
              
              var term = {};
              term.term = topicTerm;
              term.probability = parseFloat(things[t].split("_")[0]);
              row.push(term);
          }

          result.push(row);
      }
    }
    
    return result;
}