Provide a k for each ngram in the field value.
        :param num_ngrams: number of ngrams in the field value
        :return: [ k, ... ] a k value for each of num_ngrams such that the sum is exactly num_bits
174270
def ks(self, num_ngrams):
        # type (int) -> [int]
        
        if self.num_bits:
            k = int(self.num_bits / num_ngrams)
            residue = self.num_bits % num_ngrams
            return ([k + 1] * residue) + ([k] * (num_ngrams - residue))
        else:
            return [self.k if self.k else 0] * num_ngrams
1212830
def _has_ngrams(self, text_id, size):
        
        if self._conn.execute(constants.SELECT_HAS_NGRAMS_SQL,
                              [text_id, size]).fetchone() is None:
            return False
        return True
1084958
def get_num_branches(self):
        
        num = {}
        for trt, branches in itertools.groupby(
                self.branches, operator.attrgetter('trt')):
            num[trt] = sum(1 for br in branches if br.effective)
        return num
1006716
def generate_ngrams(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    if args.catalogue:
        catalogue = utils.get_catalogue(args)
    else:
        catalogue = None
    store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
187846
def get_num_paths(self):
        
        # NB: the algorithm assume a symmetric logic tree for the GSIMs;
        # in the future we may relax such assumption
        num_branches = self.get_num_branches()
        if not sum(num_branches.values()):
            return 0
        num = 1
        for val in num_branches.values():
            if val:  # the branch is effective
                num *= val
        return num
155914
static public function getFreq(array $ngrams, string $sep = ' ') : array
    {
        //getting the frequencies of the ngrams array and an array with no repetition
        $ngramsUnique = array_count_values($ngrams);

        //array to be the product of this function
        $ngramsFinal = array();

        //creates an array of tokens per ngram
        $ngramsArray = self::ngramsAsArray($sep, $ngrams);

        $ngramSize = count($ngramsArray[0]);

        $tokens_frequencies = self::readFreq($sep, $ngrams);
        $combo_frequencies  = self::readCombFreq($sep, $ngrams);

        //interate the array with no repeated ngrams
        foreach ($ngramsUnique as $ngramString => $ngramFrequency) {
            $ngramsFinal[$ngramString] = array($ngramFrequency); //putting into the final array an array of frequencies (first, the ngram frequency)

            $ngramArray = explode($sep, $ngramString); //getting an array of tokens of the ngram
            foreach ($ngramArray as $kToken => $token) { //iterating the array of tokens of the ngram
                $ngramsFinal[$ngramString][$kToken+1] = $tokens_frequencies[$token][$kToken]; //getting the frequency of the token

                if($ngramSize > 2) {
                    //getting the combined frequency of the tokens
                    for ($i = $kToken+1; $i < $ngramSize; $i++) {
                        $ngramsFinal[$ngramString][$ngramSize+$kToken+$i] = $combo_frequencies[$token.$sep.$ngramArray[$i]][$kToken][$i];
                    }
                }
            }

        }

        return $ngramsFinal;
    }
448236
def dcm(self, dcm):
        
        assert(len(dcm) == 3)
        for sub in dcm:
            assert(len(sub) == 3)

        self._dcm = np.array(dcm)

        # mark other representations as outdated, will get generated on next
        # read
        self._q = None
        self._euler = None
324125
def search(self, ngram):
		'''
		

		'''
		mask = self._document_index_mask(ngram)
		return pd.DataFrame({
			'text': self.get_texts()[mask],
			'category': [self._category_idx_store.getval(idx)
			             for idx in self._y[mask]]
		})
250735
def MP(candidate, references, n):
    
    counts = Counter(ngrams(candidate, n))
    if not counts:
        return 0

    max_counts = {}
    for reference in references:
        reference_counts = Counter(ngrams(reference, n))
        for ngram in counts:
            max_counts[ngram] = max(max_counts.get(ngram, 0), reference_counts[ngram])

    clipped_counts = dict((ngram, min(count, max_counts[ngram])) for ngram, count in counts.items())

    return sum(clipped_counts.values()) / sum(counts.values())
128220
public NGram createNGram(List words) {
    int[] nums = new int[words.size()]; //ngram needs it own copy of this array so keep making a unique one for each instance.
    for (int wi=0;wi<nums.length;wi++) {
      nums[wi] = wordMap.getIndex(words.get(wi));
      if (nums[wi] == -1) {
        return null;
      }
    }
    return new NGram(nums);
  }
147117
def get_ngrams(path):
    """"""
    with open(path, encoding='utf-8') as fh:
        ngrams = [ngram.strip() for ngram in fh.readlines()]
    return ngrams
522049
static public function odds(array $ngram, int $totalNgrams) : float
    {
        $var = self::setStatVariables($ngram, $totalNgrams);

        if ($var['RminusJ'] == 0) {
    	    $var['RminusJ'] = 1;
        }

        if ($var['LminusJ'] == 0) {
    	    $var['LminusJ'] = 1;
        }

        $term1 = $var['jointFrequency'] * $var['n22'];
        $term2 = $var['RminusJ'] * $var['LminusJ'];

        $odds = $term1/$term2;

        return $odds;
    }
166762
private final int getNumPages(long numBytes) {
		if (numBytes < 0) {
			throw new IllegalArgumentException("The number of bytes to allocate must not be negative.");
		}
		
		final long numPages = numBytes >>> this.pageSizeBits;
		if (numPages <= Integer.MAX_VALUE) {
			return (int) numPages;
		} else {
			throw new IllegalArgumentException("The given number of bytes correstponds to more than MAX_INT pages.");
		}
	}
778193
static public function ll(array $ngram, int $totalNgrams) : float
    {
        $var = self::setStatVariables($ngram, $totalNgrams);

        $logLikelihood = 0;

        if($var['jointFrequency']) {
            $logLikelihood += $var['jointFrequency'] * self::computePMI ( $var['jointFrequency'], $var['m11'] );
        }

        if($var['LminusJ']) {
            $logLikelihood += $var['LminusJ'] * self::computePMI( $var['LminusJ'], $var['m12'] );
        }

        if($var['RminusJ']) {
            $logLikelihood += $var['RminusJ']  * self::computePMI( $var['RminusJ'], $var['m21'] );
        }

        if($var['n22']) {
            $logLikelihood += $var['n22'] * self::computePMI( $var['n22'], $var['m22'] );
        }

        return $logLikelihood * 2;
    }
1317440
def num_in_memory(self):
    """"""
    n = len(self._data) - 1
    while n >= 0:
      if isinstance(self._data[n], _TensorValueDiscarded):
        break
      n -= 1
    return len(self._data) - 1 - n
951970
def ngram(n, iter_tokens):
    
    z = len(iter_tokens)
    return (iter_tokens[i:i+n] for i in range(z-n+1))
1033960
def compute_ngrams(word_list, S=3, T=3):
    
    _ngrams = []
    if isinstance(word_list, str):
        word_list = [word_list]
    for word in word_list:
        for n in range(S, T+1):
            _ngrams += zip(*(word[i:] for i in range(n)))
    return [''.join(_ngram) for _ngram in _ngrams]