/ the Sum of word occurence probabilities in log scale (a measure of how common and probable the corrected segmentation is).</returns>
504263
public SegmentedSuggestion wordSegmentation(String input, int maxEditDistance, int maxSegmentationWordLength) {
    if (input.isEmpty()) {
      return new SegmentedSuggestion();
    }
    int arraySize = Math.min(maxSegmentationWordLength, input.length());
    SegmentedSuggestion[] compositions = new SegmentedSuggestion[arraySize];
    for (int i = 0; i < arraySize; i++) {
      compositions[i] = new SegmentedSuggestion();
    }

    int circularIndex = -1;

    //outer loop (column): all possible part start positions
    for (int j = 0; j < input.length(); j++) {
      //inner loop (row): all possible part lengths (from start position): part can't be bigger than longest word in dictionary (other than long unknown word)
      int imax = Math.min(input.length() - j, maxSegmentationWordLength);
      for (int i = 1; i <= imax; i++) {
        //get top spelling correction/ed for part
        String part = input.substring(j, j + i);
        int separatorLength = 0;
        int topEd = 0;
        double topProbabilityLog;
        String topResult;

        if (Character.isWhitespace(part.charAt(0))) {
          //remove space for levensthein calculation
          part = part.substring(1);
        } else {
          //add ed+1: space did not exist, had to be inserted
          separatorLength = 1;
        }

        //remove space from part1, add number of removed spaces to topEd
        topEd += part.length();
        //remove space
        part = part.replace(" ", ""); //=System.Text.RegularExpressions.Regex.Replace(part1, @"\s+", "");
        //add number of removed spaces to ed
        topEd -= part.length();

        List<SuggestItem> results = this.lookup(part, SymSpell.Verbosity.Top, maxEditDistance);
        if (results.size() > 0) {
          topResult = results.get(0).term;
          topEd += results.get(0).distance;
          //Naive Bayes Rule
          //we assume the word probabilities of two words to be independent
          //therefore the resulting probability of the word combination is the product of the two word probabilities

          //instead of computing the product of probabilities we are computing the sum of the logarithm of probabilities
          //because the probabilities of words are about 10^-10, the product of many such small numbers could exceed (underflow) the floating number range and become zero
          //log(ab)=log(a)+log(b)
          topProbabilityLog = Math.log10((double) results.get(0).count / (double) N);
        } else {
          topResult = part;
          //default, if word not found
          //otherwise long input text would win as long unknown word (with ed=edmax+1 ), although there there should many spaces inserted
          topEd += part.length();
          topProbabilityLog = Math.log10(10.0 / (N * Math.pow(10.0, part.length())));
        }

        int destinationIndex = ((i + circularIndex) % arraySize);

        //set values in first loop
        if (j == 0) {
          compositions[destinationIndex].segmentedString = part;
          compositions[destinationIndex].correctedString = topResult;
          compositions[destinationIndex].distanceSum = topEd;
          compositions[destinationIndex].probabilityLogSum = topProbabilityLog;
        } else if ((i == maxSegmentationWordLength)
          //replace values if better probabilityLogSum, if same edit distance OR one space difference
          || (((compositions[circularIndex].distanceSum + topEd == compositions[destinationIndex].distanceSum) || (compositions[circularIndex].distanceSum + separatorLength + topEd == compositions[destinationIndex].distanceSum)) && (compositions[destinationIndex].probabilityLogSum < compositions[circularIndex].probabilityLogSum + topProbabilityLog))
          //replace values if smaller edit distance
          || (compositions[circularIndex].distanceSum + separatorLength + topEd < compositions[destinationIndex].distanceSum)) {
          compositions[destinationIndex].segmentedString = compositions[circularIndex].segmentedString + " " + part;
          compositions[destinationIndex].correctedString = compositions[circularIndex].correctedString + " " + topResult;
          compositions[destinationIndex].distanceSum = compositions[circularIndex].distanceSum + topEd;
          compositions[destinationIndex].probabilityLogSum = compositions[circularIndex].probabilityLogSum + topProbabilityLog;
        }
      }
      circularIndex++;
      if (circularIndex >= arraySize) {
        circularIndex = 0;
      }
    }
    return compositions[circularIndex];

  }
661542
private double nextIteration() {
    // compute contribution of p(a|b_i) for each feature and the new
    // correction parameter
    double loglikelihood = 0.0;
    CFMOD = 0.0;
    int numEvents = 0;
    int numCorrect = 0;
    for (TID = 0; TID < numTokens; TID++) {
      // TID, modeldistribution and PID are globals used in 
      // the updateModelExpects procedure.  They need to be set.
      eval(contexts[TID], modelDistribution);
      for (int j = 0; j < contexts[TID].length; j++) {
        PID = contexts[TID][j];
        int[] activeOutcomes = modelExpects[PID].getOutcomes();
        for (int aoi=0;aoi<activeOutcomes.length;aoi++) {
          OID = activeOutcomes[aoi];
          modelExpects[PID].updateParameter(aoi,modelDistribution[OID] * numTimesEventsSeen[TID]);
        }
        if (_useSlackParameter) {
          for (OID = 0; OID < numOutcomes; OID++) {
            if (!modelExpects[PID].contains(OID)) {
              CFMOD += modelDistribution[OID] * numTimesEventsSeen[TID];
            }
          }
        }
      }
      if (_useSlackParameter)
        CFMOD += (constant - contexts[TID].length) * numTimesEventsSeen[TID];

      loglikelihood += Math.log(modelDistribution[outcomes[TID]]) * numTimesEventsSeen[TID];
      numEvents += numTimesEventsSeen[TID];
      if (printMessages) {
        int max = 0;
        for (OID = 1; OID < numOutcomes; OID++) {
          if (modelDistribution[OID] > modelDistribution[max]) {
            max = OID;
          }
        }
        if (max == outcomes[TID]) {
          numCorrect += numTimesEventsSeen[TID];
        }
      }

    }
    display(".");

    // compute the new parameter values
    for (PID = 0; PID < numPreds; PID++) {
      double[] observed = observedExpects[PID].getParameters();
      double[] model = modelExpects[PID].getParameters();
      int[] activeOutcomes = params[PID].getOutcomes();
      for (int aoi=0;aoi<activeOutcomes.length;aoi++) {
        params[PID].updateParameter(aoi,(Math.log(observed[aoi])) - Math.log(model[aoi]));
        modelExpects[PID].setParameter(aoi,0.0); // re-initialize to 0.0's
      }
    }
    if (CFMOD > 0.0 && _useSlackParameter)
      correctionParam += (cfObservedExpect - Math.log(CFMOD));

    display(". loglikelihood=" + loglikelihood + "\t" + ((double) numCorrect / numEvents) + "\n");
    return (loglikelihood);
  }
271701
private static void checkAndCorrect(final double[] probabilities) {
		boolean ok = true;
		for (int i = probabilities.length; --i >= 0 && ok;) {
			ok = Double.isFinite(probabilities[i]);
		}

		if (!ok) {
			final double value = 1.0/probabilities.length;
			for (int i = probabilities.length; --i >= 0;) {
				probabilities[i] = value;
			}
		}
	}
1310369
def score(self, obs, pstates):
        
        pstates_idx = np.array([self.e[p] for p in pstates])
        framelogprob = self._compute_log_likelihood(obs, pstates_idx)
        logprob, _ = self._do_forward_pass(framelogprob, pstates_idx)
        return logprob
1200345
public Counter<L> logProbabilityOf(Datum<L, F> example) {
    if(example instanceof RVFDatum<?, ?>)return logProbabilityOfRVFDatum((RVFDatum<L,F>)example);
    Counter<L> scores = scoresOf(example);
    Counters.logNormalizeInPlace(scores);
    return scores;
  }
63110
def get_scores_for_category(self, cat_word_counts, not_cat_word_counts):
		'''
		
		'''
		beta = self.beta_
		assert len(cat_word_counts) == len(not_cat_word_counts)
		if self.priors_ is not None:
			p = self.priors_
			assert len(p) == len(cat_word_counts)
			precision = ((cat_word_counts + p * 1.) /
			             (cat_word_counts + not_cat_word_counts + 2 * p))
			recall = (cat_word_counts + p) * 1. / (cat_word_counts.sum() + p.sum())
		else:
			precision = (cat_word_counts * 1. / (cat_word_counts + not_cat_word_counts))
			recall = cat_word_counts * 1. / cat_word_counts.sum()
		precision_normcdf = ScaledFScore._safe_scaler(self.scaler_algo_, precision)
		recall_normcdf = ScaledFScore._safe_scaler(self.scaler_algo_, recall)
		scores = self._weighted_h_mean(precision_normcdf, recall_normcdf)
		scores[np.isnan(scores)] = 0.
		return scores
894049
double fullProbabilisticAlgorithmCardinality() {
        final int m = this.m/*for performance*/;

        // compute the "indicator function" -- sum(2^(-M[j])) where M[j] is the
        // 'j'th register value
        double sum = 0;
        int numberOfZeroes = 0/*"V" in the paper*/;
        final LongIterator iterator = probabilisticStorage.registerIterator();
        while(iterator.hasNext()) {
            final long register = iterator.next();

            sum += 1.0 / (1L << register);
            if(register == 0L) numberOfZeroes++;
        }

        // apply the estimate and correction to the indicator function
        final double estimator = alphaMSquared / sum;
        if((numberOfZeroes != 0) && (estimator < smallEstimatorCutoff)) {
            return HLLUtil.smallEstimator(m, numberOfZeroes);
        } else if(estimator <= largeEstimatorCutoff) {
            return estimator;
        } else {
            return HLLUtil.largeEstimator(log2m, regwidth, estimator);
        }
    }
957755
def _handle_outliers(self, p_o):
        
        if self.ignore_outliers:
            outliers = np.where(p_o.sum(axis=1)==0)[0]
            if outliers.size > 0:
                p_o[outliers, :] = 1.0
                self.found_outliers = True
        return p_o
977012
private Counter<L> probabilityOfRVFDatum(RVFDatum<L, F> example) {
    // NB: this duplicate method is needed so it calls the scoresOf method
    // with a RVFDatum signature
    Counter<L> scores = logProbabilityOfRVFDatum(example);
    for (L label : scores.keySet()) {
      scores.setCount(label, Math.exp(scores.getCount(label)));
    }
    return scores;
  }
703482
def log_likelihood_rankings(data, params):
    """"""
    loglik = 0
    params = np.asarray(params)
    for ranking in data:
        for i, winner in enumerate(ranking[:-1]):
            loglik -= logsumexp(params.take(ranking[i:]) - params[winner])
    return loglik
707142
def _relative_frequency(self, word):
		""""""

		count = self.type_counts.get(word, 0)
		return math.log(count/len(self.type_counts)) if count > 0 else 0
781229
def countWordOverlapFrequencies(filename="goodOverlapPairs.pkl"):
  
  with open(filename,"rb") as f:
    goodOverlapPairs = pickle.load(f)

  with open("word_bitmaps_40_bits_minimum.pkl","rb") as f:
    bitmaps = pickle.load(f)

  # Count how often each word has a highly overlapping match with other words
  wordFrequencies = {}
  for w1, w2, overlap in goodOverlapPairs:
    wordFrequencies[w1] = wordFrequencies.get(w1, 0) + 1


  printTemplate = PrettyTable(["Num High Overlaps", "Word", "On Bits"],
                              sortby="Num High Overlaps", reversesort=True)

  for word in wordFrequencies.iterkeys():
    printTemplate.add_row([wordFrequencies[word], word, len(bitmaps[word])])

  print printTemplate
1012676
private double getScore(int pos, Lexicon lexicon, double numdocs) {
		double score = 0;
		int indexTerm = this.indices[pos];
		double occurences = (double) this.freqs[pos];

		int fieldNum = this.indexToField[pos];
		double frequency = occurences / tokensPerField[fieldNum];

		// is there a custom weight for this field?
		String fieldName = lexicon.getFields()[fieldNum];
		WeightingMethod method = lexicon.getMethod(fieldName);

		if (method.equals(Parameters.WeightingMethod.BOOLEAN)) {
			score = 1;
		} else if (method.equals(Parameters.WeightingMethod.OCCURRENCES)) {
			score = occurences;
		} else if (method.equals(Parameters.WeightingMethod.FREQUENCY)) {
			score = frequency;
		} else if (method.equals(Parameters.WeightingMethod.TFIDF)) {
			int df = lexicon.getDocFreq(indexTerm);
			double idf = numdocs / (double) df;
			score = frequency * Math.log(idf);
			if (idf == 1)
				score = frequency;
		}
		return score;
	}
1039655
public String getRandomPassword(int countTotal, int countCapitals, int countSpecials) {

        String res = "";

        String[] Normals = ArrayUtils.addAll(ArrayUtils.addAll(Capitals, Letters), Numbers);
        Random rand = new Random();
        for (int i = 0; i < (countTotal - countCapitals - countSpecials); i++) {
            res = res + Normals[rand.nextInt(Normals.length)];
        }
        for (int i = 0; i < countSpecials; i++) {
            int pos = rand.nextInt(res.length());
            res = res.substring(0, pos) + Specials[rand.nextInt(Specials.length)] + res.substring(pos);
        }
        for (int i = 0; i < countCapitals; i++) {
            int pos = rand.nextInt(res.length());
            res = res.substring(0, pos) + Capitals[rand.nextInt(Capitals.length)] + res.substring(pos);
        }
        return res;
    }
83681
public double probabilityOf(E key) {
    if (counter.containsKey(key)) {
      return counter.getCount(key);
    } else {
      int remainingKeys = numberOfKeys - counter.size();
      if (remainingKeys <= 0) {
        return 0.0;
      } else {
        return (reservedMass / remainingKeys);
      }
    }
  }
1148112
@Override
  public void calculate(double[] x) {

    double prob = 0.0; // the log prob of the sequence given the model, which is the negation of value at this point
    double[][] weights = to2D(x);

    // the expectations over counts
    // first index is feature index, second index is of possible labeling
    double[][] E = empty2D();

    // iterate over all the documents
    for (int m = 0; m < data.length; m++) {
      int[][][] docData = data[m];
      int[] docLabels = labels[m];

      // make a clique tree for this document
      CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, docData, labelIndices, numClasses, classIndex, backgroundSymbol);

      // compute the log probability of the document given the model with the parameters x
      int[] given = new int[window - 1];
      Arrays.fill(given, classIndex.indexOf(backgroundSymbol));
      if (docLabels.length>docData.length) { // only true for self-training
        // fill the given array with the extra docLabels
        System.arraycopy(docLabels, 0, given, 0, given.length);
        // shift the docLabels array left
        int[] newDocLabels = new int[docData.length];
        System.arraycopy(docLabels, docLabels.length-newDocLabels.length, newDocLabels, 0, newDocLabels.length);
        docLabels = newDocLabels;
      }
      // iterate over the positions in this document
      for (int i = 0; i < docData.length; i++) {
        int label = docLabels[i];
        double p = cliqueTree.condLogProbGivenPrevious(i, label, given);
        if (VERBOSE) {
          System.err.println("P(" + label + "|" + ArrayMath.toString(given) + ")=" + p);
        }
        prob += p;
        System.arraycopy(given, 1, given, 0, given.length - 1);
        given[given.length - 1] = label;
      }

      // compute the expected counts for this document, which we will need to compute the derivative
      // iterate over the positions in this document
      for (int i = 0; i < data[m].length; i++) {
        // for each possible clique at this position
        for (int j = 0; j < data[m][i].length; j++) {
          Index<CRFLabel> labelIndex = labelIndices[j];
          // for each possible labeling for that clique
          for (int k = 0; k < labelIndex.size(); k++) {
            int[] label = labelIndex.get(k).getLabel();
            double p = cliqueTree.prob(i, label); // probability of these labels occurring in this clique with these features
            for (int n = 0; n < data[m][i][j].length; n++) {
              E[data[m][i][j][n]][k] += p;
            }
          }
        }
      }
    }

    if (Double.isNaN(prob)) { // shouldn't be the case
      throw new RuntimeException("Got NaN for prob in CRFLogConditionalObjectiveFunction.calculate()");
    }

    value = -prob;
    if(VERBOSE){
      System.err.println("value is " + value);
    }
    // compute the partial derivative for each feature by comparing expected counts to empirical counts
    int index = 0;
    for (int i = 0; i < E.length; i++) {
      for (int j = 0; j < E[i].length; j++) {
        derivative[index++] = (E[i][j] - Ehat[i][j]);
        if (VERBOSE) {
          System.err.println("deriv(" + i + "," + j + ") = " + E[i][j] + " - " + Ehat[i][j] + " = " + derivative[index - 1]);
        }
      }
    }


    // incorporate priors
    if (prior == QUADRATIC_PRIOR) {
      double sigmaSq = sigma * sigma;
      for (int i = 0; i < x.length; i++) {
        double k = 1.0;
        double w = x[i];
        value += k * w * w / 2.0 / sigmaSq;
        derivative[i] += k * w / sigmaSq;
      }
    } else if (prior == HUBER_PRIOR) {
      double sigmaSq = sigma * sigma;
      for (int i = 0; i < x.length; i++) {
        double w = x[i];
        double wabs = Math.abs(w);
        if (wabs < epsilon) {
          value += w * w / 2.0 / epsilon / sigmaSq;
          derivative[i] += w / epsilon / sigmaSq;
        } else {
          value += (wabs - epsilon / 2) / sigmaSq;
          derivative[i] += ((w < 0.0) ? -1.0 : 1.0) / sigmaSq;
        }
      }
    } else if (prior == QUARTIC_PRIOR) {
      double sigmaQu = sigma * sigma * sigma * sigma;
      for (int i = 0; i < x.length; i++) {
        double k = 1.0;
        double w = x[i];
        value += k * w * w * w * w / 2.0 / sigmaQu;
        derivative[i] += k * w / sigmaQu;
      }
    }


  }
187461
def cumulative_probabilities( self ):
        
        partition_function = np.sum( self.p )
        return np.cumsum( self.p ) / partition_function