Generate unigram, bigram, trigram ... and the max-gram,
    different from powerset(), this function will not generate skipped combinations such as (1,3)
359272
def power_ngram(iter_tokens):
    
    return chain.from_iterable(ngram(j, iter_tokens) for j in range(1, len(iter_tokens) + 1))
758721
def _combine_ngrams(ngrams, joiner) -> str:
        """"""
        if isinstance(ngrams, str):
            return ngrams
        else:
            combined = joiner.join(ngrams)
            return combined
928987
def p_Yield(p):
    '''
    
    '''
    if len(p) == 2:
        p[0] = Yield(None, None)
    elif len(p) == 3:
        p[0] = Yield(p[2], None)
    else:
        p[0] = Yield(p[2], p[4])
523376
def genslices(n):
    """"""
    def range_with_none():
        yield None
        yield from range(-n, n+1)

    for t in product(range_with_none(), range_with_none(), range_with_none()):
        s = slice(*t)
        if s.step != 0:
            yield s
906016
def possiblesplits(n, minsplits=2, maxsplits=0):
    """"""
    if not maxsplits: maxsplits = n
    for nrsplits in range(minsplits,maxsplits + 1):
        for split in sum_to_n(n,nrsplits):
            split_with_indices = []
            begin = 0
            for length in split:
                split_with_indices.append( (begin, length) )
                begin += length
            yield split_with_indices
33067
def bigram(self, items):

        

        def bigram_join(tok_list):
            text = nltk.bigrams(tok_list)
            return list(map(lambda x: x[0] + '.' + x[1], text))

        if items == "tokens":
            self.bigrams = list(map(bigram_join, self.tokens))
        elif items == "stems":
            self.bigrams = list(map(bigram_join, self.stems))
        else:
            raise ValueError("Items must be either \'tokens\' or \'stems\'.")
1103660
def ngrams(self, n=3):
    
    if n <= 0:
        return []
    grams = [WordList(self.words[i:i+n], parent=self)
                        for i in range(len(self.words) - n + 1)]
    return grams
128220
public NGram createNGram(List words) {
    int[] nums = new int[words.size()]; //ngram needs it own copy of this array so keep making a unique one for each instance.
    for (int wi=0;wi<nums.length;wi++) {
      nums[wi] = wordMap.getIndex(words.get(wi));
      if (nums[wi] == -1) {
        return null;
      }
    }
    return new NGram(nums);
  }
611218
function createCombination(array, combinationLength) {
    array = (0, _array.excludeTheSame)(array);
    if (!(0, _is.isNumber)(combinationLength)) {
        throw new Error('combinationLength should be number');
    }
    if (combinationLength < 1 || combinationLength > array.length) {
        throw new Error('combinationLength should be in the correct range');
    }

    return __createCombination(array, combinationLength);
}
1245520
private static int guessNumDistinctiveGrams(int textLength, int gramLength) {
        switch (gramLength) {
            case 1:
                return Math.min(80, textLength);
            case 2:
                if (textLength < 40) return textLength;
                if (textLength < 100) return (int)(textLength*0.8);
                if (textLength < 1000) return (int)(textLength * 0.6);
                return (int)(textLength * 0.5);
            case 3:
                if (textLength < 40) return textLength;
                if (textLength < 100) return (int)(textLength*0.9);
                if (textLength < 1000) return (int)(textLength * 0.8);
                return (int)(textLength * 0.6);
            case 4:
            case 5:
            default:
                if (textLength < 100) return textLength;
                if (textLength < 1000) return (int)(textLength * 0.95);
                return (int)(textLength * 0.9);
        }
    }
119177
private ArrayList<String> ngram(List tokens, int[] gramSizes2) {
        ArrayList<String> list = new ArrayList<String>();
        StringBuffer buf = new StringBuffer();
        for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > tokens.size())
                continue;
            for (int i = 0; i < tokens.size() - len+1; i++) {
                buf.delete(0, buf.length());
                int k = 0;
                for(; k < len-1; ++k)   {
                    buf.append(tokens.get(i+k));
                    buf.append(' ');
                }
                buf.append(tokens.get(i+k));
                list.add(buf.toString().intern());
            }
        }
        return list;
    }
879807
def combination_memo(n, r):
    """"""
    memo = {}
    def recur(n, r):
        if n == r or r == 0:
            return 1
        if (n, r) not in memo:
            memo[(n, r)] = recur(n - 1, r - 1) + recur(n - 1, r)
        return memo[(n, r)]
    return recur(n, r)
505623
def generate_skip_gram_data_set(self, token_list):
        '''
        
        '''
        n_gram_tuple_zip = self.generate_tuple_zip(token_list, 3)
        skip_gram_list = []
        for pre, point, post in n_gram_tuple_zip:
            skip_gram_list.append((point, pre))
            skip_gram_list.append((point, post))
        return zip(skip_gram_list)
277183
def rs_generator_poly_all(max_nsym, fcr=0, generator=2):
    ''''''
    g_all = {}
    g_all[0] = g_all[1] = [1]
    for nsym in xrange(max_nsym):
        g_all[nsym] = rs_generator_poly(nsym, fcr, generator)
    return g_all
881948
function computeCombinationLength(elementNumber, combinationLength) {
    elementNumber = Number(elementNumber);
    combinationLength = Number(combinationLength);
    if (!_regex.regex.positiveInteger.test(String(elementNumber))) {
        throw new Error("elementNumber should be positiveNumber but got " + elementNumber);
    }
    if (!(_regex.regex.positiveInteger.test(String(combinationLength)) || combinationLength === 0)) {
        throw new Error("elementNumber should be positiveNumber or 0 but got " + combinationLength);
    }
    if (combinationLength > elementNumber) {
        throw new Error("The max value of combinationLength should be " + elementNumber + " but got " + combinationLength);
    }
    if (combinationLength === 0) {
        return 1;
    }
    return computeFactorial(elementNumber) / (computeFactorial(combinationLength) * computeFactorial(elementNumber - combinationLength));
}
941235
def ngrams(self, with_counts=False):
        
        query = "SELECT "
        for i in reversed(range(self.cardinality)):
            if i != 0:
                query += "word_{0}, ".format(i)
            elif i == 0:
                query += "word"

        if with_counts:
            query += ", count"

        query += " FROM _{0}_gram;".format(self.cardinality)

        result = self.execute_sql(query)
        for row in result:
            yield tuple(row)
856399
private static String combinatorialPatternMaker(String[] str, int str_size, int c) {
        String comb_pattern = "";
        int curr_comb = 1;
        for (int i = 0; i < str_size; i++) {
            if ((c & curr_comb) != 0) {
                comb_pattern += str[i] + "\\s";
            } else {
                comb_pattern += "[\\w]+\\s";
            }
            curr_comb = curr_comb << 1;
        }
        //System.err.println(comb_pattern);
        return comb_pattern;
    }