Tokenization rules
341773
public function setupRules()
    {
        $this->rules->addMany([
            'string' => CommonFeatures::strings(['single' => '\'', 'double' => '"'], [
                'context' => ['!operator.escape', '!comment', '!string', '!expression'],
            ]),

            'string.heredoc' => new Rule(new RegexMatcher('/<<<\s*(\w+)\R(?P<string>.*?)\R\1;/sm', ['string' => Token::NAME, 0 => 'keyword.heredoc']), ['context' => ['!comment']]),
            'string.nowdoc'  => new Rule(new RegexMatcher('/<<<\s*\'(\w+)\'\R(?P<string>.*?)\R\1;/sm', ['string' => Token::NAME, 0 => 'keyword.nowdoc']), ['context' => ['!comment']]),

            'variable' => new Rule(new RegexMatcher('/(?:[^\\\]|^)(\$[a-z_]\w*)/i'), [
                'context' => ['*comment.docblock', '!string.nowdoc', '!string.single', '!comment']
            ]),
            'variable.property' => new Rule(new RegexMatcher('/(?=(?:\w|\)|\])\s*->([a-z_]\w*))/i'), [
                'priority' => -2,
                'context' => ['*comment.docblock', '!string.nowdoc', '!string.single', '!comment']
            ]),

            'symbol.function' => new Rule(new RegexMatcher('/function\s+([a-z_]\w+)\s*\(/i')),
            'symbol.class'    => [
                new Rule(new RegexMatcher('/(?:class|new|use|extends)\s+([\w\\\]+)/i')),
                new Rule(new RegexMatcher('/([\w\\\]+)::/i')),
                new Rule(new RegexMatcher('/@(?:var|property(?:-read|-write)?)(?:\s+|\s+\$\w+\s+)([^$][\w\\\]+)/i'), ['context' => ['comment.docblock']]),
            ],
            
            'expression.in-string' => new Rule(new RegexMatcher('/(?=(\{\$((?>[^${}]+|(?1))+)\}))/x'), [
                'context' => ['string'],
                'factory' => new TokenFactory(LanguageToken::class),
                'inject'  => $this
            ]),

            'symbol.class.interface' => [
                new Rule(new RegexMatcher('/interface\s+([\w\\\]+)/i')),
                new Rule(new DelegateRegexMatcher(
                    '/implements\s+((?:[\w\\\]+)(?:,\s*([\w\\\]+))+)/i',
                    function($match, TokenFactoryInterface $factory) {
                        foreach (preg_split('/,\s*/', $match[1][0], 0, PREG_SPLIT_OFFSET_CAPTURE) as $interface) {
                            yield $factory->create(Token::NAME, [
                                'pos' => $match[1][1] + $interface[1],
                                'length' => strlen($interface[0])]
                            );
                        }
                    }
                )),
            ],

            'symbol.namespace' => [
                /*new Rule(new RegexMatcher('/(\\\{0,2}(?:\w+\\\{1,2})+)\w+/i'), [
                    'context' => ['*symbol', '*none']
                ]),*/

                new Rule(new RegexMatcher('/namespace\s*(\\\{0,2}(?:\w+\\\{1,2})+\w+);/i'), [
                    'context' => ['*symbol', '*none']
                ]),
            ],

            'operator.escape' => [
                new Rule(new RegexMatcher('/(\\\(?:x[0-9a-fA-F]{1,2}|u\{[0-9a-fA-F]{1,6}\}|[0-7]{1,3}|[^\'\\\]))/i'), [
                    'context' => ['string.double', '!operator.escape']
                ]),
                new Rule(new RegexMatcher('/(\\\[\'\\\])/i'), [
                    'context' => ['string', '!operator.escape']
                ]),
            ],

            'comment' => new Rule(new CommentMatcher(['//', '#'], [
                '$.docblock' => ['/**', '*/'],
                ['/*', '*/']
            ]), ['priority' => 4]),

            'symbol.annotation' => new Rule(new RegexMatcher('/[\s]+(@[\w-]+)/i'), [
                'context' => ['comment.docblock']
            ]),

            'call' => new Rule(new RegexMatcher('/([a-z_]\w*)\s*\(/i'), ['priority' => -1]),

            'constant' => new Rule(new WordMatcher(array_merge([
                '__CLASS__', '__DIR__', '__FILE__', '__FUNCTION__',
                '__LINE__', '__METHOD__', '__NAMESPACE__', '__TRAIT__',
            ], array_keys(get_defined_constants(true)["Core"]))), ['priority' => -2]),
            'constant.static' => new Rule(new RegexMatcher('/(?:[\w\\\]+::|const\s+)(\w+)/i'), ['priority' => -2]),

            'keyword' => new Rule(new WordMatcher([
                '__halt_compiler', 'abstract', 'and', 'array',
                'as', 'break', 'callable', 'case', 'catch',
                'class', 'clone', 'const', 'continue', 'declare',
                'default', 'die', 'do', 'echo', 'else', 'elseif',
                'empty', 'enddeclare', 'endfor', 'endforeach', 'endif',
                'endswitch', 'endwhile', 'eval', 'exit', 'extends',
                'final', 'finally', 'for', 'foreach', 'function',
                'global', 'goto', 'if', 'implements', 'include', 'include_once',
                'instanceof', 'insteadof', 'interface', 'isset', 'list',
                'namespace', 'new', 'or', 'print', 'private', 'protected',
                'public', 'require', 'require_once', 'return', 'static',
                'switch', 'throw', 'trait', 'try', 'unset', 'parent', 'self',
                'use', 'var', 'while', 'xor', 'yield'
            ]), ['context' => ['!string', '!variable', '!comment']]),

            'keyword.cast' => new Rule(
                new RegexMatcher('/(\((?:int|integer|bool|boolean|float|double|real|string|array|object|unset)\))/')
            ),

            'delimiter' => new Rule(new RegexMatcher('/(<\?php|<\?=|\?>)/')),
            'number'    => new Rule(new RegexMatcher('/(-?(?:0[0-7]+|0[xX][0-9a-fA-F]+|0b[01]+|\d+))/')),

            'operator.punctuation' => new Rule(new WordMatcher([',', ';'], ['separated' => false]), ['priority' => 0]),
        ]);
    }
302865
function Tokenize(query) {
  var parts = __splitTrim(query, WHERE);
  var pathway = parts[0];
  var where = parts[1];

  synopsis.pathway = __splitTrim(pathway, COMMA);
  for (var i = 0, len = synopsis.pathway.length; i < len; i++) {
    synopsis.pathway[i] = __splitTrim(synopsis.pathway[i], DELIMITER);
    if (synopsis.pathway[i][0] == WILDCARD)
      synopsis.pathway[i].shift();
    if (synopsis.pathway[i].length === 0)
      synopsis.pathway.splice(i, 1);
  }

  var lastLeft = -1,
    lastRight = -1,
    current = 0;
  while (current < where.length) {
    if (where[current] === LEFT) {
      lastLeft = current;
    } else if (where[current] === RIGHT) {
      lastRight = current;
      if (lastRight > lastLeft && lastLeft !== -1) {
        var k = 'gr' + '_' + new Date().getTime();
        synopsis.groups[k] = where.substring(lastLeft + 1, lastRight);
        where = where.replace(LEFT + synopsis.groups[k] + RIGHT, k);
        current = -1;
      }
    }
    current += 1;
  }
  LogicalGrouping(AST, where);
}
997188
function parser(tokens) {
  // add Close token before each Else
  const toks = tokens.reduce((prev, tok) => {
    if (tok.tokenType === 'Else') {
      return [...prev, new Close(), tok];
    }

    return [...prev, tok];
  }, []);

  const [body] = parse(toks, '', 1);
  return body;
}
1113089
function tokenize (path) {
  const tokens = []

  const parts = path.split('.')

  for (let i = 0; i < parts.length; i++) {
    const part = parts[i].trim()

    if (part.length === 0) {
      continue
    }

    const arrayIndexes = part.split(SPLIT_REG_EXP)

    tokens.push(arrayIndexes[0])

    for (let j = 1; j < arrayIndexes.length; j++) {
      if (arrayIndexes[j].length === 0) {
        continue
      }

      tokens.push(Number(arrayIndexes[j]))
    }
  }
  return tokens
}
778837
function tokenizeText(text, callback) {
    "use strict";
    processTokens(tokenizer.tokenize(text) || [], function (error, tokens) {
        process.nextTick(function () {
            callback(error, tokens);
        });
    });
}
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
1243328
def word_tokenize(text):
    
    for (regexp, replacement) in RULES1:
        text = sub(regexp, replacement, text)
    # add extra space to make things easier
    text = " " + text + " "
    for (regexp, replacement) in RULES2:
        text = sub(regexp, replacement, text)
    for regexp in CONTRACTIONS:
        text = sub(regexp, r"\1 \2 ", text)
    # split and return
    return text.split()
854859
function Tokenizer(text) {
        this._state = TEXT;
        this._buffer = text;
        this._sectionStart = 0;
        this._sectionStartPos = {line: 0, ch: 0};
        this._index = 0;
        this._indexPos = {line: 0, ch: 0};
        this._special = 0; // 1 for script, 2 for style
        this._token = null;
        this._nextToken = null;
    }
1058109
protected function tokenize()
    {
        $operators = array(
            '(' =>  \Erebot\Styling\Parser::TK_PAR_OPEN,
            ')' =>  \Erebot\Styling\Parser::TK_PAR_CLOSE,
            '+' =>  \Erebot\Styling\Parser::TK_OP_ADD,
            '-' =>  \Erebot\Styling\Parser::TK_OP_SUB,
            '#' =>  \Erebot\Styling\Parser::TK_OP_COUNT,
        );

        while ($this->position < $this->length) {
            $c          = $this->formula[$this->position];
            $subject    = substr($this->formula, $this->position);

            // Operators ("(", ")", "+", "-" & "#").
            if (isset($operators[$c])) {
                $this->parser->doParse($operators[$c], $c);
                $this->position++;
                continue;
            }

            // Real numbers (eg. "3.14").
            if (preg_match(self::PATT_REAL, $subject, $matches)) {
                $this->position += strlen($matches[0]);
                $this->parser->doParse(
                    \Erebot\Styling\Parser::TK_NUMBER,
                    (double) $matches[0]
                );
                continue;
            }

            // Integers (eg. "42").
            if (preg_match(self::PATT_INTEGER, $subject, $matches)) {
                $this->position += strlen($matches[0]);
                $this->parser->doParse(
                    \Erebot\Styling\Parser::TK_NUMBER,
                    (int) $matches[0]
                );
                continue;
            }

            // Whitespace.
            if (strpos(" \t", $c) !== false) {
                $this->position++;
                continue;
            }

            // Variable names.
            if (preg_match(self::PATT_VAR_NAME, $subject, $matches)) {
                $this->position += strlen($matches[0]);
                $this->parser->doParse(
                    \Erebot\Styling\Parser::TK_VARIABLE,
                    $matches[0]
                );
                continue;
            }

            // Raise an exception.
            $this->parser->doParse(
                \Erebot\Styling\Parser::YY_ERROR_ACTION,
                $c
            );
        }

        // End of tokenization.
        $this->parser->doParse(0, 0);
    }
447566
function processUnary(tokenizer, scope) {
  const unaryAddSubtract = {
    '-': value => -value,
    '+': value => value
  };

  if (tokenizer.token.match(types.ADD_SUBTRACT)) {
    return createOperatorNode(unaryAddSubtract[tokenizer.token.lexeme], [processUnary(tokenizer.skipToken(), scope)]);
  }

  if (tokenizer.token.match(types.INCREASE_DECREASE) || tokenizer.token.match(types.LOGICAL_NEGATION)) {
    return createOperatorNode(tokenizer.token.value, [processUnary(tokenizer.skipToken(), scope)]);
  }

  return processIdentifiers(tokenizer, scope);
}
247551
def listified_tokenizer(source):
    """"""
    io_obj = io.StringIO(source)
    return [list(a) for a in tokenize.generate_tokens(io_obj.readline)]
197891
def _iter_tokens(self):
        """"""
        reobj, actions, nextstates = self._rules[self.states[-1]]
        mobj = reobj.match(self.string, self.pos)
        while mobj is not None:
            text = mobj.group(0)
            idx = mobj.lastindex - 1
            nextstate = nextstates[idx]

            # Take action
            actions[idx](self, text)
            while self.tokens:
                yield self.pop_token()
            if nextstate and nextstate != self.states[-1]:
                self.states[-1] = nextstate

            # Update position variables
            self.pos = mobj.end()
            lines = text.split('\n')
            nlines = len(lines) - 1
            if nlines == 0:
                self.offset = self.offset + len(lines[0])
            else:
                self.lineno = self.lineno + nlines
                self.offset = 1 + len(lines[-1])

            reobj, actions, nextstates = self._rules[self.states[-1]]
            mobj = reobj.match(self.string, self.pos)

        if self.pos != len(self.string):
            msg = "unexpected character"
            text = self.string[self.pos]
            raise RunError(msg, self.lineno, self.offset, text)

        yield EndToken("", self.lineno, self.offset)
555732
def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
        ""
        for rule in self.pre_rules: t = rule(t)
        toks = tok.tokenizer(t)
        for rule in self.post_rules: toks = rule(toks)
        return toks
465893
function tokenizer(stream,state) {
    // in multi-line string
    if (state.in_string) {
      state.in_string = (!doubleQuote(stream));
      return rval(state,stream,"string");
    }

    // in multi-line atom
    if (state.in_atom) {
      state.in_atom = (!singleQuote(stream));
      return rval(state,stream,"atom");
    }

    // whitespace
    if (stream.eatSpace()) {
      return rval(state,stream,"whitespace");
    }

    // attributes and type specs
    if (!peekToken(state) &&
        stream.match(/-\s*[a-zß-öø-ÿ][\wØ-ÞÀ-Öß-öø-ÿ]*/)) {
      if (is_member(stream.current(),typeWords)) {
        return rval(state,stream,"type");
      }else{
        return rval(state,stream,"attribute");
      }
    }

    var ch = stream.next();

    // comment
    if (ch == '%') {
      stream.skipToEnd();
      return rval(state,stream,"comment");
    }

    // colon
    if (ch == ":") {
      return rval(state,stream,"colon");
    }

    // macro
    if (ch == '?') {
      stream.eatSpace();
      stream.eatWhile(anumRE);
      return rval(state,stream,"macro");
    }

    // record
    if (ch == "#") {
      stream.eatSpace();
      stream.eatWhile(anumRE);
      return rval(state,stream,"record");
    }

    // dollar escape
    if (ch == "$") {
      if (stream.next() == "\\" && !stream.match(escapesRE)) {
        return rval(state,stream,"error");
      }
      return rval(state,stream,"number");
    }

    // dot
    if (ch == ".") {
      return rval(state,stream,"dot");
    }

    // quoted atom
    if (ch == '\'') {
      if (!(state.in_atom = (!singleQuote(stream)))) {
        if (stream.match(/\s*\/\s*[0-9]/,false)) {
          stream.match(/\s*\/\s*[0-9]/,true);
          return rval(state,stream,"fun");      // 'f'/0 style fun
        }
        if (stream.match(/\s*\(/,false) || stream.match(/\s*:/,false)) {
          return rval(state,stream,"function");
        }
      }
      return rval(state,stream,"atom");
    }

    // string
    if (ch == '"') {
      state.in_string = (!doubleQuote(stream));
      return rval(state,stream,"string");
    }

    // variable
    if (/[A-Z_Ø-ÞÀ-Ö]/.test(ch)) {
      stream.eatWhile(anumRE);
      return rval(state,stream,"variable");
    }

    // atom/keyword/BIF/function
    if (/[a-z_ß-öø-ÿ]/.test(ch)) {
      stream.eatWhile(anumRE);

      if (stream.match(/\s*\/\s*[0-9]/,false)) {
        stream.match(/\s*\/\s*[0-9]/,true);
        return rval(state,stream,"fun");      // f/0 style fun
      }

      var w = stream.current();

      if (is_member(w,keywordWords)) {
        return rval(state,stream,"keyword");
      }else if (is_member(w,operatorAtomWords)) {
        return rval(state,stream,"operator");
      }else if (stream.match(/\s*\(/,false)) {
        // 'put' and 'erlang:put' are bifs, 'foo:put' is not
        if (is_member(w,bifWords) &&
            ((peekToken(state).token != ":") ||
             (peekToken(state,2).token == "erlang"))) {
          return rval(state,stream,"builtin");
        }else if (is_member(w,guardWords)) {
          return rval(state,stream,"guard");
        }else{
          return rval(state,stream,"function");
        }
      }else if (is_member(w,operatorAtomWords)) {
        return rval(state,stream,"operator");
      }else if (lookahead(stream) == ":") {
        if (w == "erlang") {
          return rval(state,stream,"builtin");
        } else {
          return rval(state,stream,"function");
        }
      }else if (is_member(w,["true","false"])) {
        return rval(state,stream,"boolean");
      }else if (is_member(w,["true","false"])) {
        return rval(state,stream,"boolean");
      }else{
        return rval(state,stream,"atom");
      }
    }

    // number
    var digitRE      = /[0-9]/;
    var radixRE      = /[0-9a-zA-Z]/;         // 36#zZ style int
    if (digitRE.test(ch)) {
      stream.eatWhile(digitRE);
      if (stream.eat('#')) {                // 36#aZ  style integer
        if (!stream.eatWhile(radixRE)) {
          stream.backUp(1);                 //"36#" - syntax error
        }
      } else if (stream.eat('.')) {       // float
        if (!stream.eatWhile(digitRE)) {
          stream.backUp(1);        // "3." - probably end of function
        } else {
          if (stream.eat(/[eE]/)) {        // float with exponent
            if (stream.eat(/[-+]/)) {
              if (!stream.eatWhile(digitRE)) {
                stream.backUp(2);            // "2e-" - syntax error
              }
            } else {
              if (!stream.eatWhile(digitRE)) {
                stream.backUp(1);            // "2e" - syntax error
              }
            }
          }
        }
      }
      return rval(state,stream,"number");   // normal integer
    }

    // open parens
    if (nongreedy(stream,openParenRE,openParenWords)) {
      return rval(state,stream,"open_paren");
    }

    // close parens
    if (nongreedy(stream,closeParenRE,closeParenWords)) {
      return rval(state,stream,"close_paren");
    }

    // separators
    if (greedy(stream,separatorRE,separatorWords)) {
      return rval(state,stream,"separator");
    }

    // operators
    if (greedy(stream,operatorSymbolRE,operatorSymbolWords)) {
      return rval(state,stream,"operator");
    }

    return rval(state,stream,null);
  }
86031
function Tokenizer(str) {

    this.str = (str || '').toString();
    this.operatorCurrent = '';
    this.operatorExpecting = '';
    this.node = null;
    this.escaped = false;

    this.list = [];

}
207298
function tokenize_sentence(sentence) {
  var tokenized = tokenizer.tokenize(sentence);
  logger.info("tokenize_sentence: " + tokenized);
  return(tokenized);
}
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token