returns the string representation of a token.
767434
def print_token(self, token_node_index):
        """"""
        err_msg = "The given node is not a token node."
        assert isinstance(self.nodes[token_node_index], TokenNode), err_msg
        onset = self.nodes[token_node_index].onset
        offset = self.nodes[token_node_index].offset
        return self.text[onset:offset]
251757
function tokenString(stream, state) {
    var current, prev, found_var = false;
    while (!stream.eol() && (current = stream.next()) != state.pending) {
      if (current === '$' && prev != '\\' && state.pending == '"') {
        found_var = true;
        break;
      }
      prev = current;
    }
    if (found_var) {
      stream.backUp(1);
    }
    if (current == state.pending) {
      state.continueString = false;
    } else {
      state.continueString = true;
    }
    return "string";
  }
766981
private String getPatternCode(final List<Token> tokens) {
        final StringBuilder sb = new StringBuilder();
        sb.append(tokens.size());
        for (final Token token : tokens) {
            sb.append(token.getType().ordinal());
        }
        return sb.toString();
    }
556159
def _FormatOpaqueToken(self, token_data):
    
    data = ''.join(['{0:02x}'.format(byte) for byte in token_data.data])
    return {'data': data}
617132
def _parse_string(self):
        """"""
        if self._next_token() in ['{', '(']:
            field = self._parse_field()
            if field:
                self.definitions[field[0]] = field[1]
87801
protected String urlDecode(String token) {
    try {
      return URLDecoder.decode(token, "utf-8");
    }
    catch (UnsupportedEncodingException e) {
      throw new RuntimeException(e);
    }
  }
29830
def get_representation(self, prefix="", suffix="\n"):
        
        res = []
        if self.hidden:
            res.append(prefix + " - " + str(self._name)
                       + " : xxxxxxxx" + suffix)
        else:
            default = self.default
            if default is None:
                default = " - "
            a = prefix + " - "
            a += str(self._name) + " : "
            if isinstance(default, types.UnicodeType):
                a += default
            else:
                a += str(default)
            a += suffix
            res.append(a)
        return res
473536
public static String getFirstToken(String string, String token) {
        if ((string != null) && string.contains(token)) {
            return string.split(token)[0];
        }
        return string;
    }
628379
def _to_string(self):
        
        return u"{}+{}@{}".format(text_type(self.definition_id), self.BLOCK_TYPE_PREFIX, self.block_type)
25161
function getWordToken(str) {
    var word = str.split(/\s/g)[0];
    return {text: word, type: 'WORD', length: word.length};
}
740036
@SuppressWarnings("unchecked")
    public String tokenAsString(Comparable tokenAsComparable) {
        ByteBuffer bb = tokenType.decompose(tokenAsComparable);
        Token token = tokenFactory.fromByteArray(bb);
        return tokenFactory.toString(token);
    }
43410
public function getTokenString()
    {
        if (!$this->tokenString) {
            $token = $this->httpRequest()->header('Authorization');
            if (!$token) {
                $token = $this->httpRequest()->post('Authorization');
            }

            return $token;
        }

        return $this->tokenString;
    }
278931
def decode_token(token):
    
    if isinstance(token, (unicode,str)):
        return _decode_token_compact(token)

    else:
        return _decode_token_json(token)
426217
private String getToken(boolean quoted) {
		// Trim leading white spaces
		while ((i1 < i2) && (Character.isWhitespace(chars[i1]))) {
			i1++;
		}
		// Trim trailing white spaces
		while ((i2 > i1) && (Character.isWhitespace(chars[i2 - 1]))) {
			i2--;
		}
		// Strip away quotation marks if necessary
		if (quoted) {
			if (((i2 - i1) >= 2) && (chars[i1] == '"') && (chars[i2 - 1] == '"')) {
				i1++;
				i2--;
			}
		}
		String result = null;
		if (i2 > i1) {
			result = new String(chars, i1, i2 - i1);
		}
		return result;
	}
1116653
public static String typeToName(int token) {
        switch (token) {
          case ERROR:           return "ERROR";
          case EOF:             return "EOF";
          case EOL:             return "EOL";
          case ENTERWITH:       return "ENTERWITH";
          case LEAVEWITH:       return "LEAVEWITH";
          case RETURN:          return "RETURN";
          case GOTO:            return "GOTO";
          case IFEQ:            return "IFEQ";
          case IFNE:            return "IFNE";
          case SETNAME:         return "SETNAME";
          case BITOR:           return "BITOR";
          case BITXOR:          return "BITXOR";
          case BITAND:          return "BITAND";
          case EQ:              return "EQ";
          case NE:              return "NE";
          case LT:              return "LT";
          case LE:              return "LE";
          case GT:              return "GT";
          case GE:              return "GE";
          case LSH:             return "LSH";
          case RSH:             return "RSH";
          case URSH:            return "URSH";
          case ADD:             return "ADD";
          case SUB:             return "SUB";
          case MUL:             return "MUL";
          case DIV:             return "DIV";
          case MOD:             return "MOD";
          case NOT:             return "NOT";
          case BITNOT:          return "BITNOT";
          case POS:             return "POS";
          case NEG:             return "NEG";
          case NEW:             return "NEW";
          case DELPROP:         return "DELPROP";
          case TYPEOF:          return "TYPEOF";
          case GETPROP:         return "GETPROP";
          case GETPROPNOWARN:   return "GETPROPNOWARN";
          case SETPROP:         return "SETPROP";
          case GETELEM:         return "GETELEM";
          case SETELEM:         return "SETELEM";
          case CALL:            return "CALL";
          case NAME:            return "NAME";
          case NUMBER:          return "NUMBER";
          case STRING:          return "STRING";
          case NULL:            return "NULL";
          case THIS:            return "THIS";
          case FALSE:           return "FALSE";
          case TRUE:            return "TRUE";
          case SHEQ:            return "SHEQ";
          case SHNE:            return "SHNE";
          case REGEXP:          return "REGEXP";
          case BINDNAME:        return "BINDNAME";
          case THROW:           return "THROW";
          case RETHROW:         return "RETHROW";
          case IN:              return "IN";
          case INSTANCEOF:      return "INSTANCEOF";
          case LOCAL_LOAD:      return "LOCAL_LOAD";
          case GETVAR:          return "GETVAR";
          case SETVAR:          return "SETVAR";
          case CATCH_SCOPE:     return "CATCH_SCOPE";
          case ENUM_INIT_KEYS:  return "ENUM_INIT_KEYS";
          case ENUM_INIT_VALUES:return "ENUM_INIT_VALUES";
          case ENUM_INIT_ARRAY: return "ENUM_INIT_ARRAY";
          case ENUM_INIT_VALUES_IN_ORDER: return "ENUM_INIT_VALUES_IN_ORDER";
          case ENUM_NEXT:       return "ENUM_NEXT";
          case ENUM_ID:         return "ENUM_ID";
          case THISFN:          return "THISFN";
          case RETURN_RESULT:   return "RETURN_RESULT";
          case ARRAYLIT:        return "ARRAYLIT";
          case OBJECTLIT:       return "OBJECTLIT";
          case GET_REF:         return "GET_REF";
          case SET_REF:         return "SET_REF";
          case DEL_REF:         return "DEL_REF";
          case REF_CALL:        return "REF_CALL";
          case REF_SPECIAL:     return "REF_SPECIAL";
          case DEFAULTNAMESPACE:return "DEFAULTNAMESPACE";
          case ESCXMLTEXT:      return "ESCXMLTEXT";
          case ESCXMLATTR:      return "ESCXMLATTR";
          case REF_MEMBER:      return "REF_MEMBER";
          case REF_NS_MEMBER:   return "REF_NS_MEMBER";
          case REF_NAME:        return "REF_NAME";
          case REF_NS_NAME:     return "REF_NS_NAME";
          case TRY:             return "TRY";
          case SEMI:            return "SEMI";
          case LB:              return "LB";
          case RB:              return "RB";
          case LC:              return "LC";
          case RC:              return "RC";
          case LP:              return "LP";
          case RP:              return "RP";
          case COMMA:           return "COMMA";
          case ASSIGN:          return "ASSIGN";
          case ASSIGN_BITOR:    return "ASSIGN_BITOR";
          case ASSIGN_BITXOR:   return "ASSIGN_BITXOR";
          case ASSIGN_BITAND:   return "ASSIGN_BITAND";
          case ASSIGN_LSH:      return "ASSIGN_LSH";
          case ASSIGN_RSH:      return "ASSIGN_RSH";
          case ASSIGN_URSH:     return "ASSIGN_URSH";
          case ASSIGN_ADD:      return "ASSIGN_ADD";
          case ASSIGN_SUB:      return "ASSIGN_SUB";
          case ASSIGN_MUL:      return "ASSIGN_MUL";
          case ASSIGN_DIV:      return "ASSIGN_DIV";
          case ASSIGN_MOD:      return "ASSIGN_MOD";
          case HOOK:            return "HOOK";
          case COLON:           return "COLON";
          case OR:              return "OR";
          case AND:             return "AND";
          case INC:             return "INC";
          case DEC:             return "DEC";
          case DOT:             return "DOT";
          case FUNCTION:        return "FUNCTION";
          case EXPORT:          return "EXPORT";
          case IMPORT:          return "IMPORT";
          case IF:              return "IF";
          case ELSE:            return "ELSE";
          case SWITCH:          return "SWITCH";
          case CASE:            return "CASE";
          case DEFAULT:         return "DEFAULT";
          case WHILE:           return "WHILE";
          case DO:              return "DO";
          case FOR:             return "FOR";
          case BREAK:           return "BREAK";
          case CONTINUE:        return "CONTINUE";
          case VAR:             return "VAR";
          case WITH:            return "WITH";
          case CATCH:           return "CATCH";
          case FINALLY:         return "FINALLY";
          case VOID:            return "VOID";
          case RESERVED:        return "RESERVED";
          case EMPTY:           return "EMPTY";
          case BLOCK:           return "BLOCK";
          case LABEL:           return "LABEL";
          case TARGET:          return "TARGET";
          case LOOP:            return "LOOP";
          case EXPR_VOID:       return "EXPR_VOID";
          case EXPR_RESULT:     return "EXPR_RESULT";
          case JSR:             return "JSR";
          case SCRIPT:          return "SCRIPT";
          case TYPEOFNAME:      return "TYPEOFNAME";
          case USE_STACK:       return "USE_STACK";
          case SETPROP_OP:      return "SETPROP_OP";
          case SETELEM_OP:      return "SETELEM_OP";
          case LOCAL_BLOCK:     return "LOCAL_BLOCK";
          case SET_REF_OP:      return "SET_REF_OP";
          case DOTDOT:          return "DOTDOT";
          case COLONCOLON:      return "COLONCOLON";
          case XML:             return "XML";
          case DOTQUERY:        return "DOTQUERY";
          case XMLATTR:         return "XMLATTR";
          case XMLEND:          return "XMLEND";
          case TO_OBJECT:       return "TO_OBJECT";
          case TO_DOUBLE:       return "TO_DOUBLE";
          case GET:             return "GET";
          case SET:             return "SET";
          case LET:             return "LET";
          case YIELD:           return "YIELD";
          case CONST:           return "CONST";
          case SETCONST:        return "SETCONST";
          case ARRAYCOMP:       return "ARRAYCOMP";
          case WITHEXPR:        return "WITHEXPR";
          case LETEXPR:         return "LETEXPR";
          case DEBUGGER:        return "DEBUGGER";
          case COMMENT:         return "COMMENT";
          case GENEXPR:         return "GENEXPR";
          case METHOD:          return "METHOD";
          case ARROW:           return "ARROW";
        }

        // Token without name
        throw new IllegalStateException(String.valueOf(token));
    }
751099
private int getToken(String inString, int index) throws Timecode.TimecodeException {
    inString = inString.trim();
    String valid = "0123456789";
    String token = "";
    int count = 0;
    for (int i = 0; i < inString.length(); i++) {
      char current = inString.charAt(i);
      if (valid.indexOf(current) > -1) {
        token += current;
      } else {
        count++;
        if (count > index)
          break; // Found the token.
        token = ""; // Start reading the next token
      }
    }
    if (count < index || token.equals(""))
      throw new Timecode.TimecodeException("Malformed timecode '" + inString + "', can't get index=" + index);
    try {
      return Integer.parseInt(token);
    } catch (NumberFormatException ex) {
      throw new Timecode.TimecodeException("Malformed timecode '" + inString + "', '" + token + "' is not an integer");
    }
  }
599554
protected function getTokenString()
    {
        $result = Request::post('_token', '');
        if ('' == $result) {
            $result = Request::get('_token', '');
        }

        if ('' == $result) {
            $result = isset($_SERVER['X-CSRF-TOKEN']) ? $_SERVER['X-CSRF-TOKEN'] : '';
        }

        return $result;
    }