Fetch the parser token for a given character. Useful for hacks that want to inspect the mode state (say, for completion).
509067
function(mode, state, tabSize, ch) {
      var txt = this.text, stream = new StringStream(txt, tabSize);
      while (stream.pos < ch && !stream.eol()) {
        stream.start = stream.pos;
        var style = mode.token(stream, state);
      }
      return {start: stream.start,
              end: stream.pos,
              string: stream.current(),
              className: style || null,
              state: state};
    }
1159963
function toOptionToken(token) {
    if (!/^-/.test(token)) {
        token = token.split(' ')[0];
        if (!/^(\[|<)/.test(token))
            token = "[" + token + "]";
    }
    var pre = token;
    var suffix = '';
    var reqIdx = token.indexOf('<');
    var optIdx = token.indexOf('[');
    var idx = !!~reqIdx ? reqIdx : !!~optIdx ? optIdx : null;
    if (idx) {
        pre = token.slice(0, idx);
        suffix = token.slice(idx);
    }
    pre = chek_1.split(pre.trim(), constants_2.SPLIT_CHARS).join('.').replace(/\s/g, '');
    return pre + ' ' + suffix;
}
957178
private Token consumeString(char quote) {
    StringBuilder sb = new StringBuilder();
    int l = line;
    int c = column;
    Token T;
    nextChar();
    while(moreChars() && ch != quote) {
      if(ch == '\\') {
        nextChar();
        switch(ch) {
          case '"':
          case '\\':
          case '/':
            sb.append(ch);
            nextChar();
            break;
          case 'b':
            sb.append('\b');
            nextChar();
            break;
          case 'f':
            sb.append('\f');
            nextChar();
            break;
          case 'n':
            sb.append('\n');
            nextChar();
            break;
          case 'r':
            sb.append('\r');
            nextChar();
            break;
          case 't':
            sb.append('\t');
            nextChar();
            break;
          case 'u':
            nextChar();
            int u = 0;
            for(int i = 0; i < 4; i++) {
              if(isHexDigit(ch)) {
                u = u * 16 + ch - '0';
                if(ch >= 'A') { // handle hex numbers: 'A' = 65, '0' = 48. 'A'-'0' = 17, 17 - 7 = 10
                  u = u - 7;
                }
              } else {
                T = new Token(TokenType.ERROR, sb.toString(), line, column);
                nextChar();
                return T;
              }
              nextChar();
            }
            sb.append((char) u);
            break;
          default:
            T = new Token(TokenType.ERROR, sb.toString(), line, column);
            nextChar();
            return T;
        }
      } else {
        sb.append(ch);
        nextChar();
      }
    }
    if(ch == quote) {
      T = new Token(TokenType.STRING, sb.toString(), l, c);
    } else {
      T = new Token(TokenType.ERROR, sb.toString(), line, column);
    }
    nextChar();
    return T;
  }
801315
def _get_parser(description):
    """"""
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('key', help="Camellia key.")
    parser.add_argument('input_file', nargs='*',
                        help="File(s) to read as input data. If none are "
                        "provided, assume STDIN.")
    parser.add_argument('-o', '--output_file',
                        help="Output file. If not provided, assume STDOUT.")
    parser.add_argument('-l', '--keylen', type=int, default=128,
                        help="Length of 'key' in bits, must be in one of %s "
                        "(default 128)." % camcrypt.ACCEPTABLE_KEY_LENGTHS)
    parser.add_argument('-H', '--hexkey', action='store_true',
                        help="Treat 'key' as a hex string rather than binary.")

    return parser
1234082
public final Parser<Token> token() {
    return new Parser<Token>() {
      @Override boolean apply(ParseContext ctxt) {
        int begin = ctxt.getIndex();
        if (!Parser.this.apply(ctxt)) {
          return false;
        }
        int len = ctxt.getIndex() - begin;
        Token token = new Token(begin, len, ctxt.result);
        ctxt.result = token;
        return true;
      }
      @Override public String toString() {
        return Parser.this.toString();
      }
    };
  }
369406
private int parseChar(int nextChar) {
		// figure out where to put the character
		// needs to handle group levels for parsing
		// examples
		/*
		 * {\f3\froman\fcharset2\fprq2{\*\panose 05050102010706020507}Symbol;}
		 * {\f7\fswiss\fcharset0\fprq2{\*\panose 020b0604020202030204}Helv{\*\falt Arial};} <- special case!!!!
		 * {\f5\froman\fcharset0 Tahoma;}
		 * {\f6\froman\fcharset0 Arial Black;}
		 * {\info(\author name}{\company company name}}
		 * ... document text ...
		 */
	    if (this.getTokeniserState() == TOKENISER_BINARY && --binByteCount <= 0)
	    	this.setTokeniserStateNormal();
	    if (this.getTokeniserState() == TOKENISER_SKIP_BYTES && --binSkipByteCount <= 0)
	    	this.setTokeniserStateNormal();
	    return this.handleCharacter(nextChar);
	}
311000
private String parseToken(final char[] terminators) {
		char ch;
		i1 = pos;
		i2 = pos;
		while (hasChar()) {
			ch = chars[pos];
			if (isOneOf(ch, terminators)) {
				break;
			}
			i2++;
			pos++;
		}
		return getToken(false);
	}
1150506
def next_token(self):
        """"""
        while self.char is not None:

            if self.char.isspace():
                # The current character is a whitespace
                self.whitespace()
                continue

            elif self.char == '#':
                # The current character is `#`
                self.advance()
                self.comment()
                continue

            elif self.char.isalpha() or self.char == '_':
                # The current character is a letter or `_`
                return self._id()

            elif self.char == ';':
                # The current character is `;`
                self.advance()
                return Token(Nature.SEMI, ';')

            elif self.char == ',':
                # The current character is `,`
                self.advance()
                return Token(Nature.COMMA, ';')

            elif self.char.isdigit():
                # The current character is a number
                return self.number()

            elif self.char == '=' and self.peek() == '=':
                # The current character is `==`
                self.advance()
                self.advance()
                return Token(Nature.EQ, '==')

            elif self.char == '!' and self.peek() == '=':
                # The current character is `!=`
                self.advance()
                self.advance()
                return Token(Nature.NE, '!=')

            elif self.char == '<' and self.peek() == '=':
                # The current character is `<=`
                self.advance()
                self.advance()
                return Token(Nature.LE, '<=')

            elif self.char == '>' and self.peek() == '=':
                # The current character is `>=`
                self.advance()
                self.advance()
                return Token(Nature.GE, '>=')

            elif self.char == '<':
                # The current character is `<`
                self.advance()
                return Token(Nature.LT, '<')

            elif self.char == '>':
                # The current character is `>`
                self.advance()
                return Token(Nature.GT, '>')

            elif self.char == '=':
                # The current character is `=`
                self.advance()
                return Token(Nature.ASSIGN, '=')

            elif self.char == '+':
                # The current character is `+`
                self.advance()
                return Token(Nature.PLUS, '+')

            elif self.char == '-':
                # The current character is `-`
                self.advance()
                return Token(Nature.MINUS, '-')

            elif self.char == '*':
                # The current character is `*`
                self.advance()
                return Token(Nature.MUL, '*')

            elif self.char == '/' and self.peek() == '/':
                # The current character is `//`
                self.advance()
                self.advance()
                return Token(Nature.INT_DIV, '//')

            elif self.char == '/':
                # The current character is `/`
                self.advance()
                return Token(Nature.DIV, '/')

            elif self.char == '(':
                # The current character is `(`
                self.advance()
                return Token(Nature.LPAREN, '(')

            elif self.char == ')':
                # The current character is `)`
                self.advance()
                return Token(Nature.RPAREN, ')')

            elif self.char == '{':
                # The current character is `{`
                self.advance()
                return Token(Nature.LBRACKET, '{')

            elif self.char == '}':
                # The current character is `}`
                self.advance()
                return Token(Nature.RBRACKET, '}')

            else:
                # The current character is unknown
                raise LexicalError(f"Invalid character `{self.char}`.")

        # End of raw input
        return Token(Nature.EOF, None)
46383
Token parseAtom() throws ParseException {
		int ch = this.read();
		Token tok = null;
		switch (ch) {
		case T_LPAREN:
			return this.processParen();
		case T_LPAREN2:
			return this.processParen2(); // '(?:'
		case T_CONDITION:
			return this.processCondition(); // '(?('
		case T_MODIFIERS:
			return this.processModifiers(); // (?modifiers ... )
		case T_INDEPENDENT:
			return this.processIndependent();
		case T_DOT:
			this.next(); // Skips '.'
			tok = Token.token_dot;
			break;

		/**
		 * char-class ::= '[' ( '^'? range ','?)+ ']' range ::= '\d' | '\w' |
		 * '\s' | category-block | range-char | range-char '-' range-char
		 * range-char ::= '\[' | '\]' | '\\' | '\' [,-efnrtv] | bmp-code |
		 * character-2 bmp-char ::= '\' 'u' [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]
		 * [0-9a-fA-F]
		 */
		case T_LBRACKET:
			return this.parseCharacterClass(true);
		case T_SET_OPERATIONS:
			return this.parseSetOperations();

		case T_BACKSOLIDUS:
			switch (this.chardata) {
			case 'd':
			case 'D':
			case 'w':
			case 'W':
			case 's':
			case 'S':
				tok = this.getTokenForShorthand(this.chardata);
				this.next();
				return tok;

			case 'e':
			case 'f':
			case 'n':
			case 'r':
			case 't':
			case 'u':
			case 'v':
			case 'x': {
				int ch2 = this.decodeEscaped();
				if (ch2 < 0x10000) {
					tok = Token.createChar(ch2);
				} else {
					tok = Token.createString(REUtil.decomposeToSurrogates(ch2));
				}
			}
				break;

			case 'c':
				return this.processBacksolidus_c();
			case 'C':
				return this.processBacksolidus_C();
			case 'i':
				return this.processBacksolidus_i();
			case 'I':
				return this.processBacksolidus_I();
			case 'g':
				return this.processBacksolidus_g();
			case 'X':
				return this.processBacksolidus_X();
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
				return this.processBackreference();

			case 'P':
			case 'p':
				int pstart = this.offset;
				tok = processBacksolidus_pP(this.chardata);
				if (tok == null)
					throw this.ex("parser.atom.5", pstart);
				break;

			default:
				tok = Token.createChar(this.chardata);
			}
			this.next();
			break;

		case T_CHAR:
			if (this.chardata == ']' || this.chardata == '{'
					|| this.chardata == '}')
				throw this.ex("parser.atom.4", this.offset - 1);
			tok = Token.createChar(this.chardata);
			int high = this.chardata;
			this.next();
			if (REUtil.isHighSurrogate(high) && this.read() == T_CHAR
					&& REUtil.isLowSurrogate(this.chardata)) {
				char[] sur = new char[2];
				sur[0] = (char) high;
				sur[1] = (char) this.chardata;
				tok = Token.createParen(Token.createString(new String(sur)), 0);
				this.next();
			}
			break;

		default:
			throw this.ex("parser.atom.4", this.offset - 1);
		}
		return tok;
	}
491255
def parse_raw_token(self, raw_token):
        """"""
        if raw_token is None:
            return (None, None, None)
        qs = parse_qs(raw_token)
        token = qs.get('oauth_token', [None])[0]
        token_secret = qs.get('oauth_token_secret', [None])[0]
        return (token, token_secret, None)
707829
def _peek_unicode(
        self, is_long
    ):  # type: (bool) -> Tuple[Optional[str], Optional[str]]
        
        # we always want to restore after exiting this scope
        with self._state(save_marker=True, restore=True):
            if self._current not in {"u", "U"}:
                raise self.parse_error(
                    InternalParserError, "_peek_unicode() entered on non-unicode value"
                )

            self.inc()  # Dropping prefix
            self.mark()

            if is_long:
                chars = 8
            else:
                chars = 4

            if not self.inc_n(chars):
                value, extracted = None, None
            else:
                extracted = self.extract()

                if extracted[0].lower() == "d" and extracted[1].strip("01234567"):
                    return None, None

                try:
                    value = chr(int(extracted, 16))
                except ValueError:
                    value = None

            return value, extracted
231713
def get_dict(parser, token):
    
    bits = token.contents.split(' ')
    return GetDict(bits[1], bits[2], ((len(bits) > 3) and bits[3]) or '', ((len(bits) > 4) and bits[4]) or '', ((len(bits) > 5) and bits[5]) or '')
1058906
def _get_token_type(self, char):
        
        if char in '()':
            return self.SPLIT, 0
        elif char == ',':
            return self.SPLIT, 1
        elif char in '<>':
            return self.IGNORE, 2
        elif char == '.':
            return self.JOIN, 3
        elif char.isdigit():
            return self.JOIN, 4
        elif char.isalpha():
            return self.JOIN, 5
        elif char == '^':
            return self.JOIN, 6
        elif char.isspace():
            return self.IGNORE, 7
        else:
            raise ValueError('TokeniserException: "{}" can not be tokenised'.format(char))
762941
def parse_string(self):
        """"""
        aliased_value = LITERAL_ALIASES.get(self.current_token.value.lower())
        if aliased_value is not None:
            return aliased_value
        return String(self.current_token.value)
1127531
private static int readingState(char[] nchar, DDLStatement retval) {

        if ( ! Character.isUnicodeIdentifierPart(nchar[0])) {
            char prev = prevChar(retval.statement);
            /* Since we only have access to the current character and the characters we have seen so far,
             * we can check for the token only after its completed and then look if it matches the required token. */
            if (checkForNextBegin) {
                if (prev == 'n' || prev == 'N') {
                    if (SQLLexer.matchToken(retval.statement, retval.statement.length() - 5, "begin") ) {
                        inAsBegin = true;
                    }
                }
                checkForNextBegin = false;
            }
            if (prev == 'd' || prev == 'D') {
                if (SQLLexer.matchToken(retval.statement, retval.statement.length() - 3, "end") ) {
                    if (inCaseWhen > 0) {
                        inCaseWhen--;
                    } else {
                        // we can terminate BEGIN ... END for multi stmt proc
                        // after all CASE ... END stmts are completed
                        inAsBegin = false;
                    }
                }
            } else if (prev == 'e' || prev == 'E') {
                if (SQLLexer.matchToken(retval.statement, retval.statement.length() - 4, "case") ) {
                    inCaseWhen++;
                }
            } else if (prev == 's' || prev == 'S') {
                if (SQLLexer.matchToken(retval.statement, retval.statement.length() - 2, "as") ) {
                    checkForNextBegin = true;
                }
            }
        }
        if (nchar[0] == '-') {
            // remember that a possible '--' is being examined
            return kStateReadingCommentDelim;
        }
        else if (nchar[0] == '\n') {
            // normalize newlines to spaces
            retval.endLineNo += 1;
            retval.statement += " ";
        }
        else if (nchar[0] == '\r') {
            // ignore carriage returns
        }
        else if (nchar[0] == ';') {
            // end of the statement
            retval.statement += nchar[0];
            // statement completed only if outside of begin..end
            if(!inAsBegin) {
                return kStateCompleteStatement;
            }
        }
        else if (nchar[0] == '\'') {
            retval.statement += nchar[0];
            return kStateReadingStringLiteral;
        }
        else if (SQLLexer.isBlockDelimiter(nchar[0])) {
            // we may be examining ### code block delimiters
            retval.statement += nchar[0];
            return kStateReadingCodeBlockDelim;
        }
        else {
            // accumulate and continue
            retval.statement += nchar[0];
        }

        return kStateReading;
    }
1277621
function decodeToken (tokenStr) {
  if (!tokenStr || tokenStr === '') return null
  try {
    return jwtDecode(tokenStr)
  } catch (error) {
    logger.error({
      description: 'Error decoding token.',
      error, func: 'decodeToken', file: 'token'
    })
    return null
  }
}
783427
private void checkToken(String s) {
		for (int i = 0; i < s.length(); i++) {
			char ch = s.charAt(i);
			if (!TOKEN.get(ch)) {
				throw new IllegalArgumentException("Invalid token character '" + ch + "' in token \"" + s + "\"");
			}
		}
	}