Takes an identifier token, and tranforms its type to
 match Ruby keywords where the identifier is actually a keyword.
 Reserved words are defined in S.8.5.1 of the Ruby spec.
4202
def translate_keyword_tokens( token )
      keywords = ["__LINE__", "__ENCODING__", "__FILE__", "BEGIN",   
                  "END", "alias", "and", "begin", "break", "case",
                  "class", "def", "defined?", "do", "else", "elsif",
                  "end", "ensure", "false", "for", "if", "in",
                  "module", "next", "nil", "not", "or", "redo",
                  "rescue", "retry", "return", "self", "super",
                  "then", "true", "undef", "unless", "until", 
                  "when", "while", "yield"]
      if keywords.include?( token.content )
        token.type = token.content.downcase.to_sym
        # Change the state if we match a keyword
        @expr_state = :beg
      end
      
      # A couple of exceptions      
      if token.content == "BEGIN"
        token.type = :begin_global 
        @expr_state = :beg
      elsif token.content == "END"
        token.type = :end_global 
        @expr_state = :beg
      end
      
      token
    end
124752
def classify(self, token_type, value, lineno, column, line):
        """"""
        if token_type == self.grammar.KEYWORD_TOKEN:
            label_index = self.grammar.keyword_ids.get(value, -1)
            if label_index != -1:
                return label_index
        label_index = self.grammar.token_ids.get(token_type, -1)
        if label_index == -1:
            raise ParseError("invalid token", token_type, value, lineno, column,
                             line)
        return label_index
486132
def _make_reserved_tokens_re(reserved_tokens):
  """"""
  if not reserved_tokens:
    return None
  escaped_tokens = [_re_escape(rt) for rt in reserved_tokens]
  pattern = "(%s)" % "|".join(escaped_tokens)
  reserved_tokens_re = _re_compile(pattern)
  return reserved_tokens_re
1087489
function processIdentifiers(tokenizer, scope) {
  let params = [];

  if (tokenizer.token.match(types.IDENTIFIER)) {
    const keys = [tokenizer.token.value];

    tokenizer.skipToken();

    while (tokenizer.token.match(types.DOT, types.OPEN_SQUARE_BRACKET)) {
      const token = tokenizer.token;

      tokenizer.skipToken();

      const isValidKey = token.match(types.OPEN_SQUARE_BRACKET) ?
        tokenizer.token.match(types.CONSTANT) :
        (
          (tokenizer.token.match(types.CONSTANT) && (typeof tokenizer.token.value === 'number')) ||
          tokenizer.token.match(types.IDENTIFIER)
        );

      if (!isValidKey) {
        throw new SyntaxError('Unexpected object key notation');
      }

      keys.push(tokenizer.token.value);
      tokenizer.skipToken();

      if (token.match(types.OPEN_SQUARE_BRACKET)) {
        if (!tokenizer.token.match(types.CLOSE_SQUARE_BRACKET)) {
          throw new SyntaxError('Closing square bracket expected');
        }

        tokenizer.skipToken();
      }
    }

    const identifier = keys.join('.');

    if (!tokenizer.token.match(types.OPEN_PARENTHESES)) {
      scope.identifiers.push(identifier);

      return createIdentifierNode(identifier);
    }

    params = [];
    tokenizer.skipToken();
    if (!tokenizer.token.match(types.CLOSE_PARENTHESES)) {
      params.push(processLogicalOr(tokenizer, scope));

      while (tokenizer.token.match(types.DELIMITER)) {
        params.push(processLogicalOr(tokenizer.skipToken(), scope));
      }
    }

    if (!tokenizer.token.match(types.CLOSE_PARENTHESES)) {
      throw new SyntaxError('Unexpected end of expression');
    }

    tokenizer.skipToken();
    return createOperatorNode(identifier, params);
  }

  return processConstants(tokenizer, scope);
}
149062
def _convert_token(self, token):
        
        # Beware that ADAL returns a pointer to its own dict, do
        # NOT change it in place
        token = token.copy()

        # If it's from ADAL, expiresOn will be in ISO form.
        # Bring it back to float, using expiresIn
        if "expiresOn" in token and "expiresIn" in token:
            token["expiresOn"] = token['expiresIn'] + time.time()
        return {self._case.sub(r'\1_\2', k).lower(): v
                for k, v in token.items()}
125550
def get_keywords_as_string(keywords):
        
        if type(keywords) is list:
            keywords = ', '.join(keywords)
        if type(keywords) is str:
            final_keywords = keywords
        elif type(keywords) is unicode:
            final_keywords = keywords.encode('utf-8')
        elif keywords is None:
            final_keywords = ""
        else:
            raise TypeError("keywords argument must be a string or a list of strings; got a %s" % type(keywords))
        return final_keywords
1095662
def obfuscatable_class(tokens, index, **kwargs):
    
    tok = tokens[index]
    token_type = tok[0]
    token_string = tok[1]
    if index > 0:
        prev_tok = tokens[index-1]
    else: # Pretend it's a newline (for simplicity)
        prev_tok = (54, '\n', (1, 1), (1, 2), '#\n')
    prev_tok_string = prev_tok[1]
    if token_type != tokenize.NAME:
        return None # Skip this token
    if token_string.startswith('__'): # Don't mess with specials
        return None
    if prev_tok_string == "class":
        return token_string
814472
def parse_prefix(identifier):
    
    pf, id = (), identifier
    if "|" in identifier:
        pf, id = tuple(identifier.split('|')[:-1]), identifier.split('|')[-1]

    return pf, id
301973
function getTokenType(match) {
  const [offset, text] = match.slice(-2);
  const token = matchToToken(match);

  if (token.type === "name") {
    if (esutils.keyword.isReservedWordES6(token.value)) {
      return "keyword";
    }

    if (
      JSX_TAG.test(token.value) &&
      (text[offset - 1] === "<" || text.substr(offset - 2, 2) == "</")
    ) {
      return "jsx_tag";
    }

    if (token.value[0] !== token.value[0].toLowerCase()) {
      return "capitalized";
    }
  }

  if (token.type === "punctuator" && BRACKET.test(token.value)) {
    return "bracket";
  }

  if (
    token.type === "invalid" &&
    (token.value === "@" || token.value === "#")
  ) {
    return "punctuator";
  }

  return token.type;
}
823656
function isIdentifier() {
            if (token() === 69 /* Identifier */) {
                return true;
            }
            // If we have a 'yield' keyword, and we're in the [yield] context, then 'yield' is
            // considered a keyword and is not an identifier.
            if (token() === 114 /* YieldKeyword */ && inYieldContext()) {
                return false;
            }
            // If we have a 'await' keyword, and we're in the [Await] context, then 'await' is
            // considered a keyword and is not an identifier.
            if (token() === 119 /* AwaitKeyword */ && inAwaitContext()) {
                return false;
            }
            return token() > 105 /* LastReservedWord */;
        }
884612
def t_ID(self, t):
        '

        if t.value[0] == '~':
            t.type = 'TYVAR'
            t.value = t.value[1:]
        elif t.value in self.reserved_words:
            t.type = self.reserved_words[t.value]
        else:
            t.type = 'ID'

        return t
282741
def keyword(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == "symbol" and token.value.lower() == expected:
        return TokenMatch(None, token.value, (token,))
1127940
def t_INDENTIFIER(t):
    '
    if t.value in reserved:
        t.type = t.value.upper()
        if t.value in reservedMap:
            t.value = reservedMap[t.value]
    elif t.value in strStatment:
        t.type = 'STATEMENT'
    return t
497689
function Parser (customKeywordSpec) {
    var keywordSpec = customKeywordSpec || {
        _: [0],
        gettext: [0],
        ngettext: [0, 1]
    };
    var openings = ['<%', '{\\*', '{{'];
    var closures = ['%>', '\\*}', '}}'];

    if (typeof keywordSpec !== 'object') {
        throw 'Invalid keyword spec';
    }

    this.keywordSpec = keywordSpec;
    this.expressionPattern = new RegExp([
        '(?:' + openings.join('[=-]|') + '[=-]) *',
        '(' + Object.keys(keywordSpec).map(escapeRegExp).join('|') + ')',
        '\\(',
        '([\\s\\S]*?)',
        '\\)',
        ' *(?:' + closures.join('|') + ')'
    ].join(''), 'g');
}
491375
function getIdentifiersFromTokens (tokens) {
  const identifiers = {}

  /**
   * Predicate to determine if token is an identifier.
   *
   * @private
   * @param {Object} token
   * @return {Boolean}
   */

  const isIdentifier = token => 'Identifier' == token.type

  /**
   * Mark token as a function identifier.
   *
   * @private
   * @param {Object} token
   * @param {Number} index
   * @return {Object} token
   */

  const markFunction = (token, index) => {
    const next = tokens[index + 1] || null
    token.isFunction = ( 'Identifier' == token.type
                        && 'object' == typeof next && next
                        && 'Punctuator' == next.type
                        && '(' == next.value
                          ? true : false )
    return token
  }

  /**
   * Mark token as a object identifier.
   *
   * @private
   * @param {Object} token
   * @param {Number} index
   * @return {Object} token
   */

  const markObject = (token, index) => {
    const next = tokens[index + 1] || null
    token.isObject = ( 'Identifier' == token.type
                      && 'object' == typeof next && next
                      && 'Punctuator' == next.type
                      && '.' == next.value
                        ? true : false )
    return token
  }

  /**
   * Assign token value to identifierss map.
   *
   * @private
   * @param {Object} map
   * @param {Object} token
   * @return {Object} map
   */

  const assign = (map, token) => {
    const value = token.value
    if (token.isFunction)
      map[value] = _ => ''
    else if (token.isObject)
      map[value] = {}
    else
      map[value] = ''
    return map
  }

  // resolve identifierss and return map
  return ( tokens
          .map((t, i) => markFunction(t, i))
          .map((t, i) => markObject(t, i))
          .filter(t => isIdentifier(t))
          .reduce((map, t) => assign(map, t), identifiers) )
}
941979
function parseKeyword(context: ParserContext): ?Token {
  return util.matchWord(context, context.language.keywords, "keyword");
}
1159171
public Token getMatchedToken(final int tokenType) throws RecognitionException {
        Token result = parser.getCurrentToken();
        boolean isIdentifierCompatible = false;
        if (identifierTokenIndex == tokenType && identifierTokenIndex > result.getType()) {
            isIdentifierCompatible = true;
        }
        if (result.getType() == tokenType || isIdentifierCompatible) {
            if (Token.EOF != tokenType && isIdentifierCompatible && result instanceof CommonToken) {
                ((CommonToken) result).setType(identifierTokenIndex);
            }
            parser.getErrorHandler().reportMatch(parser);
            parser.consume();
        } else {
            result = parser.getErrorHandler().recoverInline(parser);
            if (parser.getBuildParseTree() && -1 == result.getTokenIndex()) {
                parser.getContext().addErrorNode(parser.createErrorNode(parser.getContext(), result));
            }
        }
        return result;
    }