Decode a list of tokens to a unicode string.

  Args:
    tokens: a list of Unicode strings
  Returns:
    a unicode string
6008
def decode(tokens):
  
  token_is_alnum = [t[0] in _ALPHANUMERIC_CHAR_SET for t in tokens]
  ret = []
  for i, token in enumerate(tokens):
    if i > 0 and token_is_alnum[i - 1] and token_is_alnum[i]:
      ret.append(u" ")
    ret.append(token)
  return "".join(ret)
906998
def t_TEXT(self, token):
        '
        text = token.lexer.lexmatch.group("text").decode("utf8")
        token.value = text
        return token
263339
def recover_codes(tokens: Iterator[Tokenizer]) -> str:
    

    tokens = iter(tokens)

    s = []

    try:
        head = next(tokens)

    except StopIteration:
        return s

    append = s.append
    lineno = head.lineno

    start_indent = colno = head.colno

    append(head.value)

    colno += len(s[-1])

    for each in tokens:
        n = each.lineno - lineno

        if n:
            append('\n' * n)
            lineno = each.lineno
            colno = each.colno

            if colno - start_indent > 0:
                append(' ' * colno)

        else:
            c = each.colno - colno
            if c:
                colno = each.colno
                if c > 0:
                    append(' ' * c)

        append(each.value)

        colno += len(s[-1])

    return s
1023475
public static void normalizeTokens(final List<List<Token>> tokens,
      final String lang) {
    for (final List<Token> sentence : tokens) {
      Normalizer.convertNonCanonicalStrings(sentence, lang);
      Normalizer.normalizeQuotes(sentence, lang);
      Normalizer.normalizeDoubleQuotes(sentence, lang);
    }
  }
231713
def get_dict(parser, token):
    
    bits = token.contents.split(' ')
    return GetDict(bits[1], bits[2], ((len(bits) > 3) and bits[3]) or '', ((len(bits) > 4) and bits[4]) or '', ((len(bits) > 5) and bits[5]) or '')
986070
def _simple_po_to_dict(tokens) -> BaseAbundance:
    
    dsl = FUNC_TO_DSL.get(tokens[FUNCTION])
    if dsl is None:
        raise ValueError('invalid tokens: {}'.format(tokens))

    return dsl(
        namespace=tokens[NAMESPACE],
        name=tokens.get(NAME),
        identifier=tokens.get(IDENTIFIER),
    )
1175073
def _decode_token(cls, token):
        
        missing_padding = len(token) % 4
        if missing_padding != 0:
            token += '=' * (4 - missing_padding)
        return json.loads(base64.b64decode(token).decode('utf-8'))
917356
def tokenize(self, s):
        
        return [s[start:end] for start, end in self.span_tokenize(s)]
149062
def _convert_token(self, token):
        
        # Beware that ADAL returns a pointer to its own dict, do
        # NOT change it in place
        token = token.copy()

        # If it's from ADAL, expiresOn will be in ISO form.
        # Bring it back to float, using expiresIn
        if "expiresOn" in token and "expiresIn" in token:
            token["expiresOn"] = token['expiresIn'] + time.time()
        return {self._case.sub(r'\1_\2', k).lower(): v
                for k, v in token.items()}
486635
def deep_unicode(s, encodings=None):
    """"""
    if encodings is None:
        encodings = ['utf-8', 'latin-1']
    if isinstance(s, (list, tuple)):
        return [deep_unicode(i) for i in s]
    if isinstance(s, dict):
        return dict([
            (deep_unicode(key), deep_unicode(s[key]))
            for key in s
        ])
        # in_dict = {}
        # for key in s:
        #     in_dict[to_unicode(key)] = to_unicode(s[key])
        # return in_dict
    elif isinstance(s, str):
        for encoding in encodings:
            try:
                return s.decode(encoding)
            except:
                pass
    return s
399808
def process_tokens(self, tokens):
        
        for tok_type, token, (start_row, start_col), _, _ in tokens:
            if tok_type == tokenize.STRING:
                # 'token' is the whole un-parsed token; we can look at the start
                # of it to see whether it's a raw or unicode string etc.
                self._process_string_token(token, start_row, start_col)
238620
def tokens2ids(tokens: Iterable[str], vocab: Dict[str, int]) -> List[int]:
    
    return [vocab.get(w, vocab[C.UNK_SYMBOL]) for w in tokens]
1253577
def decode(self, encoded):
        
        encoded = super().decode(encoded)
        tokens = [self.itos[index] for index in encoded]
        return self.detokenize(tokens)
1338714
def _tokens_to_subtoken(self, tokens):
        
        ret = []
        for token in tokens:
            ret.extend(
                self._escaped_token_to_subtoken_strings(_escape_token(token, self._alphabet)))
        return ret
1364673
def parse_tokens(self, tokens):
        
        if len(tokens) == 1:
            return list(), tokens, {"kubectl": self.ast.help}
        else:
            tokens.reverse()
        parsed, unparsed, suggestions = self.treewalk(self.ast, parsed=list(), unparsed=tokens)
        if not suggestions and unparsed:
            # TODO: @vogxn: This is hack until we include expected value types for each option and argument.
            # Whenver we recieve no suggestions but are left with unparsed tokens we pop the value and walk the
            # tree again without values
            logger.debug("unparsed tokens remain, possible value encountered")
            unparsed.pop()
            parsed.reverse()
            unparsed.extend(parsed)
            logger.debug("resuming treewalk with tokens: %s", unparsed)
            return self.treewalk(self.ast, parsed=list(), unparsed=unparsed)
        else:
            return parsed, unparsed, suggestions
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]