token = [CFWS] 1*ttext [CFWS]

    The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or
    tspecials.  We also exclude tabs even though the RFC doesn't.

    The RFC implies the CFWS but is not explicit about it in the BNF.
761714
def get_token(value):
    
    mtoken = Token()
    if value and value[0] in CFWS_LEADER:
        token, value = get_cfws(value)
        mtoken.append(token)
    if value and value[0] in TOKEN_ENDS:
        raise errors.HeaderParseError(
            "expected token but found '{}'".format(value))
    token, value = get_ttext(value)
    mtoken.append(token)
    if value and value[0] in CFWS_LEADER:
        token, value = get_cfws(value)
        mtoken.append(token)
    return mtoken, value
273770
def ansiformat(attr, text):
    
    result = []
    if attr[:1] == attr[-1:] == '+':
        result.append(codes['blink'])
        attr = attr[1:-1]
    if attr[:1] == attr[-1:] == '*':
        result.append(codes['bold'])
        attr = attr[1:-1]
    if attr[:1] == attr[-1:] == '_':
        result.append(codes['underline'])
        attr = attr[1:-1]
    result.append(codes[attr])
    result.append(text)
    result.append(codes['reset'])
    return ''.join(result)
316829
def _suffix_rules(token, tag="NN"):
    
    if isinstance(token, (list, tuple)):
        token, tag = token
    if token.endswith("ing"):
        tag = "VBG"
    if token.endswith("ly"):
        tag = "RB"
    if token.endswith("s") and not token.endswith(("is", "ous", "ss")):
        tag = "NNS"
    if token.endswith(("able", "al", "ful", "ible", "ient", "ish", "ive", "less", "tic", "ous")) or "-" in token:
        tag = "JJ"
    if token.endswith("ed"):
        tag = "VBN"
    if token.endswith(("ate", "ify", "ise", "ize")):
        tag = "VBP"
    return [token, tag]
150051
def pcfgtokensofw(self, word):
        
        tok = helper.tokens(word)

        sym = ['__{0}{1}__'.format(helper.whatchar(w), len(w))
               for w in tok]
        S = ['__S__' + ''.join(sym).replace('_', '') + '__']
        return S + sym + tok
812563
private void consumeCRLF(ImapRequestLineReader request)
            throws ProtocolException {
        char next = request.nextChar();
        if (next != '\n') {
            consumeChar(request, '\r');
        }
        consumeChar(request, '\n');
    }
907072
function vchar() {
        return wrap('vchar', compareToken(function vcharFunc(tok) {
            var code = tok.charCodeAt(0);
            var accept = (0x21 <= code && code <= 0x7E);
            if (opts.rfc6532) {
                accept = accept || isUTF8NonAscii(tok);
            }
            return accept;
        }));
    }
277812
private HttpTokens.Token next(ByteBuffer buffer) {
        byte ch = buffer.get();

        HttpTokens.Token t = HttpTokens.TOKENS[0xff & ch];

        if (DEBUG)
            LOG.debug("token={}", t);

        switch (t.getType()) {
            case CNTL:
                throw new IllegalCharacterException(_state, t, buffer);

            case LF:
                _cr = false;
                break;

            case CR:
                if (_cr)
                    throw new BadMessageException("Bad EOL");

                _cr = true;
                return null;

            case ALPHA:
            case DIGIT:
            case TCHAR:
            case VCHAR:
            case HTAB:
            case SPACE:
            case OTEXT:
            case COLON:
                if (_cr)
                    throw new BadMessageException("Bad EOL");
                break;

            default:
                break;
        }

        return t;
    }
360438
function markSC(tokens) {
    let tokensLength = tokens.length;
    let ws = -1; // Flag for whitespaces
    let sc = -1; // Flag for whitespaces and comments
    let t; // Current token

    // For every token in the token list, mark spaces and line breaks
    // as spaces (set both `ws` and `sc` flags). Mark multiline comments
    // with `sc` flag.
    // If there are several spaces or tabs or line breaks or multiline
    // comments in a row, group them: take the last one's index number
    // and save it to the first token in the group as a reference:
    // e.g., `ws_last = 7` for a group of whitespaces or `sc_last = 9`
    // for a group of whitespaces and comments.
    for (var i = 0; i < tokensLength; i++) {
      t = tokens[i];
      switch (t.type) {
        case TokenType.Space:
        case TokenType.Tab:
          t.ws = true;
          t.sc = true;

          if (ws === -1) ws = i;
          if (sc === -1) sc = i;

          break;
        case TokenType.Newline:
          t.ws = true;
          t.sc = true;

          ws = ws === -1 ? i : ws;
          sc = sc === -1 ? i : ws;

          tokens[ws].ws_last = i - 1;
          tokens[sc].sc_last = i - 1;
          tokens[i].ws_last = i;
          tokens[i].sc_last = i;

          ws = -1;
          sc = -1;

          break;
        case TokenType.CommentML:
        case TokenType.CommentSL:
          if (ws !== -1) {
            tokens[ws].ws_last = i - 1;
            ws = -1;
          }

          t.sc = true;

          break;
        default:
          if (ws !== -1) {
            tokens[ws].ws_last = i - 1;
            ws = -1;
          }

          if (sc !== -1) {
            tokens[sc].sc_last = i - 1;
            sc = -1;
          }
      }
    }

    if (ws !== -1) tokens[ws].ws_last = i - 1;
    if (sc !== -1) tokens[sc].sc_last = i - 1;
  }
1091794
private static String readToken(Buffer buffer) {
    try {
      long tokenSize = buffer.indexOfElement(TOKEN_DELIMITERS);
      if (tokenSize == -1L) tokenSize = buffer.size();

      return tokenSize != 0L
          ? buffer.readUtf8(tokenSize)
          : null;
    } catch (EOFException e) {
      throw new AssertionError();
    }
  }
176558
private static int matchesAt(CharSequence text, int offsetInText, CharSequence substring) {
        int len = substring.length();
        int textLength = text.length();
        if (textLength + offsetInText > len) {
            return -1;
        }
        int i = 0;
        for (int j = offsetInText; i < len; ++i, ++j) {
            char pc = substring.charAt(i);
            char tc = text.charAt(j);
            if (pc != tc) return -1;
        }
        return i;
    }
1114555
function qtext() {
        return wrap('qtext', or(
            function qtextFunc1() {
                return compareToken(function qtextFunc2(tok) {
                    var code = tok.charCodeAt(0);
                    var accept =
                        (33 === code) ||
                        (35 <= code && code <= 91) ||
                        (93 <= code && code <= 126);
                    if (opts.rfc6532) {
                        accept = accept || isUTF8NonAscii(tok);
                    }
                    return accept;
                });
            },
            obsQtext
        )());
    }
341096
def tokhex(length=10, urlsafe=False):
    
    if urlsafe is True:
        return secrets.token_urlsafe(length)
    return secrets.token_hex(length)
975736
protected static String cfdefToString(CfDef cfDef) throws IOException
    {
        assert cfDef != null;
        // this is so awful it's kind of cool!
        TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
        try
        {
            return Hex.bytesToHex(serializer.serialize(cfDef));
        }
        catch (TException e)
        {
            throw new IOException(e);
        }
    }
564201
def t_QUOTED_STRING(t):
    '
    # TODO: Add support for:
    # - An undefined escape sequence (such as "\a" in a context where "a"
    # has no special meaning) is interpreted as if there were no backslash
    # (in this case, "\a" is just "a"), though that may be changed by
    # extensions.
    # - Non-printing characters such as tabs, CRLF, and control characters
    # are permitted in quoted strings.  Quoted strings MAY span multiple
    # lines.  An unencoded NUL (US-ASCII 0) is not allowed in strings.
    t.value = t.value.strip('"').replace(r'\"', '"').replace(r'\\', '\\')
    return t
615982
function getToken() {
  tokenType = TOKENTYPE.NULL;
  token = '';

  // skip over whitespaces: space, tab, newline, and carriage return
  while (c === ' ' || c === '\t' || c === '\n' || c === '\r') {
    next();
  }

  // check for delimiters
  if (DELIMITERS[c]) {
    tokenType = TOKENTYPE.DELIMITER;
    token = c;
    next();
    return;
  }

  // check for a number
  if (isDigit(c) || c === '-') {
    tokenType = TOKENTYPE.NUMBER;

    if (c === '-') {
      token += c;
      next();

      if (!isDigit(c)) {
        throw createSyntaxError('Invalid number, digit expected', index);
      }
    }
    else if (c === '0') {
      token += c;
      next();
    }
    else {
      // digit 1-9, nothing extra to do
    }

    while (isDigit(c)) {
      token += c;
      next();
    }

    if (c === '.') {
      token += c;
      next();

      if (!isDigit(c)) {
        throw createSyntaxError('Invalid number, digit expected', index);
      }

      while (isDigit(c)) {
        token += c;
        next();
      }
    }

    if (c === 'e' || c === 'E') {
      token += c;
      next();

      if (c === '+' || c === '-') {
        token += c;
        next();
      }

      if (!isDigit(c)) {
        throw createSyntaxError('Invalid number, digit expected', index);
      }

      while (isDigit(c)) {
        token += c;
        next();
      }
    }

    return;
  }

  // check for a string
  if (c === '"') {
    tokenType = TOKENTYPE.STRING;
    next();

    while (c !== '' && c !== '"') {

      if (c === '\\') {
        // handle escape characters
        next();

        let unescaped = ESCAPE_CHARACTERS[c];
        if (unescaped !== undefined) {
          token += unescaped;
          next();
        }
        else if (c === 'u') {
          // parse escaped unicode character, like '\\u260E'
          next();

          let hex = '';
          for (let u = 0; u < 4; u++) {
            if (!isHex(c)) {
              throw createSyntaxError('Invalid unicode character');
            }
            hex += c;
            next();
          }

          token += String.fromCharCode(parseInt(hex, 16));
        }
        else {
          throw createSyntaxError('Invalid escape character "\\' + c + '"', index);
        }
      }
      else {
        // a regular character
        token += c;
        next();
      }
    }

    if (c !== '"') {
      throw createSyntaxError('End of string expected');
    }
    next();

    return;
  }

  // check for symbols (true, false, null)
  if (isAlpha(c)) {
    tokenType = TOKENTYPE.SYMBOL;

    while (isAlpha(c)) {
      token += c;
      next();
    }

    return;
  }

  // something unknown is found, wrong characters -> a syntax error
  tokenType = TOKENTYPE.UNKNOWN;
  while (c !== '') {
    token += c;
    next();
  }
  throw createSyntaxError('Syntax error in part "' + token + '"');
}
534570
def _symbol_token_end(c, ctx, is_field_name, value=None):
    """"""
    if value is None:
        value = ctx.value
    if is_field_name or c in _SYMBOL_TOKEN_TERMINATORS or ctx.quoted_text:
        # This might be an annotation or a field name. Mark it as self-delimiting because a symbol token termination
        # character has been found.
        ctx.set_self_delimiting(ctx.quoted_text).set_pending_symbol(value).set_quoted_text(False)
        trans = ctx.immediate_transition(ctx.whence)
    else:
        trans = ctx.event_transition(IonEvent, IonEventType.SCALAR, IonType.SYMBOL, _as_symbol(value))
    return trans
419620
public static boolean isText(ByteBuf buf, Charset charset) {
        return isText(buf, buf.readerIndex(), buf.readableBytes(), charset);
    }