Creates a token set of all tokens in the index using `lunr.TokenSet`
776365
def _create_token_set(self):
        
        self.token_set = TokenSet.from_list(sorted(list(self.inverted_index.keys())))
1318153
public final String[] getTokensWithMultiWords(final String[] tokens) {
    final Span[] multiWordSpans = multiWordsToSpans(tokens);
    final List<String> tokenList = new ArrayList<String>(Arrays.asList(tokens));
    int counter = 0;
    for (final Span mwSpan : multiWordSpans) {
      final int fromIndex = mwSpan.getStart() - counter;
      final int toIndex = mwSpan.getEnd() - counter;
      // System.err.println(fromIndex + " " + toIndex);
      // add to the counter the length of the sublist removed
      // to allow the fromIndex and toIndex to match wrt to the tokenList
      // indexes
      counter = counter + tokenList.subList(fromIndex, toIndex).size() - 1;
      // create the multiword joining the sublist
      final String multiWord = Joiner.on("#").join(
          tokenList.subList(fromIndex, toIndex));
      // remove the sublist containing the tokens to be replaced in the span
      tokenList.subList(fromIndex, toIndex).clear();
      // add the multiword containing the tokens in one Span
      tokenList.add(fromIndex, multiWord);
    }
    return tokenList.toArray(new String[tokenList.size()]);
  }
593603
def _subtoken_ids_to_tokens(self, subtokens):
    """"""
    escaped_tokens = "".join([
        self.subtoken_list[s] for s in subtokens
        if s < len(self.subtoken_list)])
    escaped_tokens = escaped_tokens.split("_")

    # All tokens in the vocabulary list have been escaped (see _escape_token())
    # so each token must be unescaped when decoding.
    ret = []
    for token in escaped_tokens:
      if token:
        ret.append(_unescape_token(token))
    return ret
379749
def tokens(self):
        
        for subsegment_or_token in self:
            if isinstance(subsegment_or_token, Segment):
                subsegment = subsegment_or_token
                for token in subsegment.tokens():
                    yield token
            else:
                token = subsegment_or_token
                yield token
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
596543
def lset(self, name, index, value):
        ""
        return self.execute_command('LSET', name, index, value)
35232
def _indices(lexer):
    """"""
    first = _expect_token(lexer, {IntegerToken}).value
    rest = _zom_index(lexer)
    return (first, ) + rest
686235
def insert_in_next_line(tokens, index, string):
    
    tokenized_string = token_utils.listified_tokenizer(string)
    for i, tok in list(enumerate(tokens[index:])):
        token_type = tok[0]
        if token_type in [tokenize.NL, tokenize.NEWLINE]:
            for count, item in enumerate(tokenized_string):
                tokens.insert(index+count+i+1, item)
            break
629177
public static List<TokensLine> convert(List<TokenEntry> tokens) {
    List<TokensLine> result = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    int startLine = Integer.MIN_VALUE;
    int startIndex = 0;
    int currentIndex = 0;
    for (TokenEntry token : tokens) {
      if (token != TokenEntry.EOF) {
        String value = token.getValue();
        int line = token.getBeginLine();
        if (line != startLine) {
          addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
          startIndex = currentIndex + 1;
          startLine = line;
        }
        currentIndex++;
        sb.append(value);
      }
    }
    addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
    return result;
  }
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
436370
def _createIndexRti(self, index, nodeName):
        
        return PandasIndexRti(index=index, nodeName=nodeName, fileName=self.fileName,
                              iconColor=self._iconColor)
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
129898
private String createSpan(final String[] tokens, final int from,
      final int to) {
    String tokenSpan = "";
    for (int i = from; i < to; i++) {
      tokenSpan += tokens[i] + " ";
    }
    tokenSpan += tokens[to];
    return tokenSpan;
  }
778472
private void findTokens() {
        mergedTokens = new TokenList();
        variable = null;
        state = TokenizerState.START;
        index = -1;

        while (index < rawTokens.length) {
            switch (state) {
            case START:
                next(TokenizerState.OUTSIDE_VARIABLE_CONTEXT);
                break;
            case OUTSIDE_VARIABLE_CONTEXT:
                handleTokenOutsideOfVariableContext();
                break;
            case INSIDE_VARIABLE_CONTEXT:
                handleTokenInsideOfVariableContext();
                break;
            default:
                break;
            }
        }
    }
935959
def _token_counts_from_generator(generator, max_chars, reserved_tokens):
  """"""
  reserved_tokens = list(reserved_tokens) + [_UNDERSCORE_REPLACEMENT]
  tokenizer = text_encoder.Tokenizer(
      alphanum_only=False, reserved_tokens=reserved_tokens)
  num_chars = 0
  token_counts = collections.defaultdict(int)
  for s in generator:
    s = tf.compat.as_text(s)
    if max_chars and (num_chars + len(s)) >= max_chars:
      s = s[:(max_chars - num_chars)]
    tokens = tokenizer.tokenize(s)
    tokens = _prepare_tokens_for_encode(tokens)
    for t in tokens:
      token_counts[t] += 1
    if max_chars:
      num_chars += len(s)
      if num_chars > max_chars:
        break
  return token_counts
67433
private IndexedSet<T> createFromIndices(IntSet indx) {
		return new IndexedSet<T>(itemToIndex, indexToItem, indx);
	}
1205287
function squashTokens (tokens) {
    var squashedTokens = [];

    var token, lastToken;
    for (var i = 0, numTokens = tokens.length; i < numTokens; ++i) {
      token = tokens[i];

      if (token) {
        if (token[0] === 'text' && lastToken && lastToken[0] === 'text') {
          lastToken[1] += token[1];
          lastToken[3] = token[3];
        } else {
          squashedTokens.push(token);
          lastToken = token;
        }
      }
    }

    return squashedTokens;
  }