Generate Ngrams from the tokens
@param array $tokens
@param int $nGramSize
@return  array return an array of the ngrams
514830
static public function create(array $tokens, $nGramSize = self::BIGRAM, $separator = ' ') : array
    {
        $separatorLength = strlen($separator);
        $length = count($tokens) - $nGramSize + 1;
        if($length < 1) {
            return [];
        }
        $ngrams = array_fill(0, $length, ''); // initialize the array

        for($index = 0; $index < $length; $index++)
        {
            for($jindex = 0; $jindex < $nGramSize; $jindex++)
            {
                $ngrams[$index] .= $tokens[$index + $jindex];
                //alterado a condição, pois não considera-se o tamanho do separador e sim a posição do ponteiro em relação ao tamanho do Ngram
                if($jindex < $nGramSize - 1) {
                    $ngrams[$index] .= $separator;
                }
            }
        }
       return $ngrams;
    }
1153714
def to_ngrams(self, terms):
        ''' 
        '''
        if len(terms) <= self.n:
            return terms
        if self.n == 1:
            n_grams = [[term] for term in terms]
        else:
            n_grams = []
            for i in range(0,len(terms)-self.n+1):
                n_grams.append(terms[i:i+self.n])
        return n_grams
867013
private function createResult(array $tokens, $requestMethod, $responseCode, array $content = null) {
        if($responseCode >= 400 && $responseCode < 600) return [];
        if ($requestMethod === HttpMethods::DELETE)     return [];
        $result = $requestMethod === HttpMethods::GET ? SelectResult::create($tokens, $content) : $content;
        krsort($result);

        return $result;
    }
1006716
def generate_ngrams(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    if args.catalogue:
        catalogue = utils.get_catalogue(args)
    else:
        catalogue = None
    store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
503731
public function parse(array $tokens)
    {
        $parser = new Parser($this->symbolContainer);

        if ($this->customGrammarChecker !== null) {
            $parser->setCustomGrammarChecker($this->customGrammarChecker);
        }

        $rootNode = $parser->parse($tokens);

        return $rootNode;
    }
119177
private ArrayList<String> ngram(List tokens, int[] gramSizes2) {
        ArrayList<String> list = new ArrayList<String>();
        StringBuffer buf = new StringBuffer();
        for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > tokens.size())
                continue;
            for (int i = 0; i < tokens.size() - len+1; i++) {
                buf.delete(0, buf.length());
                int k = 0;
                for(; k < len-1; ++k)   {
                    buf.append(tokens.get(i+k));
                    buf.append(' ');
                }
                buf.append(tokens.get(i+k));
                list.add(buf.toString().intern());
            }
        }
        return list;
    }
608547
protected function filterTokens(array $tokens)
    {
        $filteredTokens = array_filter($tokens, function (Token $t) {
            return $t->getType() !== TokenType::Whitespace;
        });

        // Return the array values only, because array_filter preserves the keys
        return array_values($filteredTokens);
    }
3831
public function generateToken($token_length=30, $prefix='') {
        $token_length = $token_length - strlen($prefix);
        if ($token_length < 0) { throw new Exception("Prefix is too long", 1); }
        
        $token = "";
        while (($len = strlen($token)) < $token_length) {
            $remaining_len = $token_length - $len;
            $bytes = random_bytes($remaining_len);
            $token .= substr(str_replace(['/', '+', '='], '', base64_encode($bytes)), 0, $remaining_len);
        }

        return $prefix.$token;
    }
455153
private function parseAggregations(\DOMXPath $xpath, \DOMNode $rootNode)
    {
        $aggs = [];

        foreach ($xpath->query(self::AGGREGATIONS_PATH, $rootNode) as $aggNode) {
            $bucketName   = $aggNode->getAttribute('name');
            $bucketConfig = [];

            foreach ($aggNode->attributes as $attribute) {
                $bucketConfig[$attribute->name] = $attribute->value;
            }
            $aggFilters = $this->parseAggregationFilters($xpath, $aggNode);
            if (!empty($aggFilters)) {
                $bucketConfig['filters'] = $aggFilters;
            }
            $bucketConfig['childBuckets'] = $this->parseAggregations($xpath, $aggNode);
            $bucketConfig['metrics'] = $this->parseMetrics($xpath, $aggNode);
            $bucketConfig['pipelines'] = $this->parsePipelines($xpath, $aggNode);
            $aggs[$bucketName] = $bucketConfig;
        }

        return $aggs;
    }
1372778
private static void ngramOnCharacter(String data, int[] gramSizes, Collection<String> list) {
    	data = data.replaceAll("\\s+", "");
    	for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > data.length())
                continue;
            for (int i = 0; i < data.length() - len; i++) {
                list.add(data.substring(i, i + len));
            }
        }
    }
1006406
public function each($batchSize = 100)
    {
        return new BatchDataIterator($this->getConnection(), [
            'qs' => $this,
            'batchSize' => $batchSize,
            'each' => true,
            'asArray' => $this->asArray,
        ]);
    }
314848
def word_tokenize(text, stopwords=_stopwords, ngrams=None, min_length=0, ignore_numeric=True):
    
    if ngrams is None:
        ngrams = 1

    text = re.sub(re.compile('\'s'), '', text)  # Simple heuristic
    text = re.sub(_re_punctuation, '', text)

    matched_tokens = re.findall(_re_token, text.lower())
    for tokens in get_ngrams(matched_tokens, ngrams):
        for i in range(len(tokens)):
            tokens[i] = tokens[i].strip(punctuation)

            if len(tokens[i]) < min_length or tokens[i] in stopwords:
                break
            if ignore_numeric and isnumeric(tokens[i]):
                break
        else:
            yield tuple(tokens)
794867
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['ORDER'])) return [];

        $headers = [];
        $orderQueryArr = array_map(function($order){
            $query = end($order['no_quotes']['parts']);
            if (empty($query)) return null;
            return !isset($order['direction']) ? $query : $query . ' ' . $order['direction'];
        },$tokens['ORDER']);
        array_push($headers, 'Order: '.implode(',',$orderQueryArr));
        return $headers;
    }
1089945
public function getCollocations()
    {
        $nGramTokens = ngrams($this->tokens, $this->nGramSize);
        return freq_dist($nGramTokens)->getKeyValuesByFrequency();
    }
1060705
private function parse($tokens, ?TypeContext $context = null): array
    {
        $count = count($tokens);
        $tagCount = 0;
        $tags = [];

        for ($i = 1; $i < $count; $i += 2) {
            $tags[] = $this->tagFactory->create($tokens[$i], $context);
            $tokens[$i] = '%' . ++$tagCount . '$s';
        }

        //In order to allow "literal" inline tags, the otherwise invalid
        //sequence "{@}" is changed to "@", and "{}" is changed to "}".
        //"%" is escaped to "%%" because of vsprintf.
        //See unit tests for examples.
        for ($i = 0; $i < $count; $i += 2) {
            $tokens[$i] = str_replace(['{@}', '{}', '%'], ['@', '}', '%%'], $tokens[$i]);
        }

        return [implode('', $tokens), $tags];
    }
1348401
public function bufferSize($bufferSize = null)
    {
        if (!func_num_args($bufferSize)) {
            return $this->_bufferSize;
        }
        return $this->_bufferSize = $bufferSize;
    }
359272
def power_ngram(iter_tokens):
    
    return chain.from_iterable(ngram(j, iter_tokens) for j in range(1, len(iter_tokens) + 1))