Calculates n-grams.

  Args:
    n: which n-grams to calculate
    text: An array of tokens

  Returns:
    A set of n-grams
769124
def _get_ngrams(n, text):
  
  ngram_set = set()
  text_length = len(text)
  max_index_ngram_start = text_length - n
  for i in range(max_index_ngram_start + 1):
    ngram_set.add(tuple(text[i:i + n]))
  return ngram_set
441680
def tokenize(self):

        

        self.tokens = []
        self.terms = OrderedDict()

        # Generate tokens.
        for token in utils.tokenize(self.text):

            # Ignore stopwords.
            if token['unstemmed'] in self.stopwords:
                self.tokens.append(None)

            else:

                # Token:
                self.tokens.append(token)

                # Term:
                offsets = self.terms.setdefault(token['stemmed'], [])
                offsets.append(token['offset'])
667394
def get_ggt(self, n, u):
        
        gk = self[0].einsum_sequence([n, u, n, u])
        result = -(2*gk*np.outer(u, u) + self[0].einsum_sequence([n, n])
                   + self[1].einsum_sequence([n, u, n, u])) / (2*gk)
        return result
685376
function grams(value) {
    var nGrams = []
    var index

    if (value === null || value === undefined) {
      return nGrams
    }

    value = value.slice ? value : String(value)
    index = value.length - n + 1

    if (index < 1) {
      return nGrams
    }

    while (index--) {
      nGrams[index] = value.slice(index, index + n)
    }

    return nGrams
  }
119177
private ArrayList<String> ngram(List tokens, int[] gramSizes2) {
        ArrayList<String> list = new ArrayList<String>();
        StringBuffer buf = new StringBuffer();
        for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > tokens.size())
                continue;
            for (int i = 0; i < tokens.size() - len+1; i++) {
                buf.delete(0, buf.length());
                int k = 0;
                for(; k < len-1; ++k)   {
                    buf.append(tokens.get(i+k));
                    buf.append(' ');
                }
                buf.append(tokens.get(i+k));
                list.add(buf.toString().intern());
            }
        }
        return list;
    }
1259900
def train_token(self, word, count):
        
        if word not in self.tokens:
            self.tokens[word] = 0

        self.tokens[word] += count
        self.tally += count
1274522
def generate_ngram_data_set(self, token_list, n=2):
        '''
        
        '''
        n_gram_tuple_zip = self.generate_tuple_zip(token_list, n)
        n_gram_tuple_list = [n_gram_tuple for n_gram_tuple in n_gram_tuple_zip]
        n_gram_data_set = self.generate_tuple_zip(n_gram_tuple_list, 2)
        return n_gram_data_set
1314629
def _nth(arr, n):
    
    try:
        return arr.iloc[n]
    except (KeyError, IndexError):
        return np.nan
902216
def chunks(l, n):
    
    newn = int(1.0 * len(l) / n + 0.5)
    for i in range(0, n - 1):
        yield l[i * newn:i * newn + newn]
    yield l[n * newn - newn:]
299129
def word_matches(s1, s2, n=3):
    
    return __matches(s1, s2, word_ngrams, n=n)
906892
def _tokenize(self, text):
        
        sentence_tokenizer = TokenizeSentence('latin')
        sentences = sentence_tokenizer.tokenize_sentences(text.lower())

        sent_words = []
        punkt = PunktLanguageVars()
        for sentence in sentences:
            words = punkt.word_tokenize(sentence)

            assert isinstance(words, list)
            words_new = []
            for word in words:
                if word not in self.punctuation or self.abbreviations or self.numbers or self.abbreviations:  # pylint: disable=line-too-long
                    words_new.append(word)

            # rm all numbers here with: re.compose(r'[09]')
            sent_words.append(words_new)

        return sent_words
256665
def sum(self, weights=None):
        
        if weights is None:
            weights = self.data.weights
        return utils.bincount(self.labels, weights, self.N)
951970
def ngram(n, iter_tokens):
    
    z = len(iter_tokens)
    return (iter_tokens[i:i+n] for i in range(z-n+1))
1065844
def read(self, n):
        
        pos = self._read_pos
        self._read_pos = min(len(self.raw), pos + n)
        return self.raw[pos:self._read_pos]
242230
def load_text(self, text, tokenizer=None):
        
        if tokenizer:
            words = [x.lower() for x in tokenizer(text)]
        else:
            words = self.tokenize(text)

        self._dictionary.update(words)
        self._update_dictionary()
239248
def _ReadN(self, n):
    
    ret = ""
    while True:
      chunk = self._read_file.read(n - len(ret))
      ret += chunk

      if len(ret) == n or not chunk:
        return ret
334810
@Nullable
    public String get(int n) {
        if (capitalword_) return null;
        int len = grams_.length(); 
        if (n < 1 || n > N_GRAM || len < n) return null;
        if (n == 1) {
            char ch = grams_.charAt(len - 1);
            if (ch == ' ') return null;
            return Character.toString(ch);
        } else {
            return grams_.substring(len - n, len);
        }
    }