so we use getTokenizer() and simple ignore the LT tokens.
358615
static List<GoogleToken> getGoogleTokens(String sentence, boolean addStartToken, Tokenizer wordTokenizer) {
    List<GoogleToken> result = new ArrayList<>();
    if (addStartToken) {
      result.add(new GoogleToken(LanguageModel.GOOGLE_SENTENCE_START, 0, 0));
    }
    List<String> tokens = wordTokenizer.tokenize(sentence);
    int startPos = 0;
    for (String token : tokens) {
      if (!StringTools.isWhitespace(token)) {
        result.add(new GoogleToken(token, startPos, startPos+token.length()));
      }
      startPos += token.length();
    }
    return result;
  }
1035322
public static String cleanseToken(String token) {
    if (token == null || token.length() == 0) { return token; };

    String cleansed = token.replaceAll(SPACE, UNDERSCORE);
    cleansed = cleansed.replaceAll(Constants.SEP, UNDERSCORE);

    return cleansed;
  }
565675
def _skip_newlines(self):
        """"""
        while self._cur_token['type'] is TT.lbreak and not self._finished:
            self._increment()
479161
private Set<TokenDefinition> extractHiddenAndIgnoredTokensFromGrammar() {
	Set<TokenDefinition> hiddenAndIgnoredTokens = new LinkedHashSet<TokenDefinition>();
	for (TokenDefinition tokenDefinition : grammar.getTokenDefinitions().getDefinitions()) {
	    Visibility visibility = tokenDefinition.getVisibility();
	    if ((visibility == Visibility.HIDDEN) || (visibility == Visibility.IGNORED)) {
		hiddenAndIgnoredTokens.add(tokenDefinition);
	    }
	}
	return hiddenAndIgnoredTokens;
    }
778472
private void findTokens() {
        mergedTokens = new TokenList();
        variable = null;
        state = TokenizerState.START;
        index = -1;

        while (index < rawTokens.length) {
            switch (state) {
            case START:
                next(TokenizerState.OUTSIDE_VARIABLE_CONTEXT);
                break;
            case OUTSIDE_VARIABLE_CONTEXT:
                handleTokenOutsideOfVariableContext();
                break;
            case INSIDE_VARIABLE_CONTEXT:
                handleTokenInsideOfVariableContext();
                break;
            default:
                break;
            }
        }
    }
989352
function(source) {
			// transform tokens
			var tokens = this.lex(source), pos = 0, token;
			for (var i = 0, il = tokens.length; i < il; i++) {
				token = tokens[i];
				token.start = pos;
				token.end = (pos += token.value.length);
			}
			return tokens;
		}
1149931
protected StreamTokenizer getStreamTokenizer(Reader reader) {
        final StreamTokenizer streamTokenizer = new StreamTokenizer(reader);
        streamTokenizer.resetSyntax();
        streamTokenizer.wordChars('a', 'z');
        streamTokenizer.wordChars('A', 'Z');
        streamTokenizer.wordChars('-', '-');
        streamTokenizer.wordChars('_', '_');
        streamTokenizer.wordChars('0', '9');
        streamTokenizer.wordChars('á', ' ');
        streamTokenizer.whitespaceChars(0, ' ');
        streamTokenizer.quoteChar('"');
        streamTokenizer.eolIsSignificant(true);
        streamTokenizer.ordinaryChar('(');
        streamTokenizer.ordinaryChar(')');
        return streamTokenizer;
    }
576908
private Map<String, ParseTreeNode> getTokens() throws GrammarException,
	    TreeException {
	Map<String, ParseTreeNode> tokens = new HashMap<String, ParseTreeNode>();
	ParseTreeNode tokensTree = parserTree.getChild("Tokens");
	ParseTreeNode tokenDefinitions = tokensTree.getChild("TokenDefinitions");
	for (ParseTreeNode tokenDefinition : tokenDefinitions
		.getChildren("TokenDefinition")) {
	    // identifier...
	    String identifier = tokenDefinition.getChild("IDENTIFIER")
		    .getText();
	    // token parts...
	    tokens.put(identifier, tokenDefinition);
	    // visibility...
	    ParseTreeNode visibilityAST = tokenDefinition.getChild("Visibility");
	    if (visibilityAST.hasChild("HIDE")) {
		tokenVisibility.put(identifier, Visibility.HIDDEN);
	    } else if (visibilityAST.hasChild("IGNORE")) {
		tokenVisibility.put(identifier, Visibility.IGNORED);
	    } else {
		tokenVisibility.put(identifier, Visibility.VISIBLE);
	    }
	}
	return tokens;
    }
1046960
protected AbstractTokenizer generateTokenizer() {
        Class<? extends AbstractTokenizer> tokenizer = parameters.getTokenizer();
        if(tokenizer==null) {
            return null;
        }

        try {
            return tokenizer.newInstance();
        } 
        catch (InstantiationException | IllegalAccessException ex) {
            throw new RuntimeException(ex);
        }
    }
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
559611
def _SkipFieldValue(tokenizer):
  
  # String/bytes tokens can come in multiple adjacent string literals.
  # If we can consume one, consume as many as we can.
  if tokenizer.TryConsumeByteString():
    while tokenizer.TryConsumeByteString():
      pass
    return

  if (not tokenizer.TryConsumeIdentifier() and
      not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and
      not tokenizer.TryConsumeFloat()):
    raise ParseError('Invalid field value: ' + tokenizer.token)
1329160
def tokenlist(item, sep, suppress=True):
    """"""
    if suppress:
        sep = sep.suppress()
    return item + ZeroOrMore(sep + item) + Optional(sep)
1214491
public void skipAll(final TokenType... tokenTypes) {
        Set<TokenType> tokenTypeSet = Sets.newHashSet(tokenTypes);
        while (tokenTypeSet.contains(lexer.getCurrentToken().getType())) {
            lexer.nextToken();
        }
    }
1225142
def filter_nonspellcheckable_tokens(line, block_out_regexes=None):
    
    all_block_out_regexes = [
        r"[^\s]*:[^\s]*[/\\][^\s]*",
        r"[^\s]*[/\\][^\s]*",
        r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+\b"
    ] + (block_out_regexes or list())

    for block_regex in all_block_out_regexes:
        for marker in re.finditer(block_regex, line):
            spaces = " " * (marker.end() - marker.start())
            line = line[:marker.start()] + spaces + line[marker.end():]

    return line
519749
public String getToken(){
		if(this.tokenPosition>0){
			String[] ar = StringUtils.split(this.line, null, this.tokenPosition+1);
			if(ar!=null && ar.length>(this.tokenPosition-1)){
				return StringUtils.trim(ar[this.tokenPosition-1]);
			}
		}
		return null;
	}
1217652
def _skip_saveframe(self, lexer):
        
        token = u""
        while token != u"save_":
            token = next(lexer)
618078
private int nextToken(StreamTokenizer tokeniser, Reader in, boolean ignoreEOF) throws IOException, ParserException {
        int token = tokeniser.nextToken();
        if (!ignoreEOF && token == StreamTokenizer.TT_EOF) {
            throw new ParserException("Unexpected end of file", getLineNumber(tokeniser, in));
        }
        return token;
    }