Convert each token or token pair to its Excel 'ptg' equivalent.
343989
def parse_tokens(tokens)
    parse_str   = ''
    last_type   = ''
    modifier    = ''
    num_args    = 0
    _class      = 0
    _classary   = [1]
    args        = tokens.dup
    # A note about the class modifiers used below. In general the class,
    # "reference" or "value", of a function is applied to all of its operands.
    # However, in certain circumstances the operands can have mixed classes,
    # e.g. =VLOOKUP with external references. These will eventually be dealt
    # with by the parser. However, as a workaround the class type of a token
    # can be changed via the repeat_formula interface. Thus, a _ref2d token can
    # be changed by the user to _ref2dA or _ref2dR to change its token class.
    #
    while (!args.empty?)
      token = args.shift

      if (token == '_arg')
        num_args = args.shift
      elsif (token == '_class')
        token = args.shift
        _class = @functions[token][2]
        # If _class is undef then it means that the function isn't valid.
        exit "Unknown function #{token}() in formula\n" if _class.nil?
        _classary.push(_class)
      elsif (token == '_vol')
        parse_str += convert_volatile()
      elsif (token == 'ptgBool')
        token = args.shift
        parse_str += convert_bool(token)
      elsif (token == '_num')
        token = args.shift
        parse_str += convert_number(token)
      elsif (token == '_str')
        token = args.shift
        parse_str += convert_string(token)
      elsif (token =~ /^_ref2d/)
        modifier  = token.sub(/_ref2d/, '')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_ref2d(token, _class)
      elsif (token =~ /^_ref3d/)
        modifier  = token.sub(/_ref3d/,'')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_ref3d(token, _class)
      elsif (token =~ /^_range2d/)
        modifier  = token.sub(/_range2d/,'')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_range2d(token, _class)
      elsif (token =~ /^_range3d/)
        modifier  = token.sub(/_range3d/,'')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_range3d(token, _class)
      elsif (token =~ /^_name/)
        modifier = token.sub(/_name/, '')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_name(token, _class)
      elsif (token == '_func')
        token = args.shift
        parse_str += convert_function(token, num_args.to_i)
        _classary.pop
        num_args = 0 # Reset after use
      elsif @ptg[token]
        parse_str += [@ptg[token]].pack("C")
      else
        # Unrecognised token
        return nil
      end
    end

    parse_str
  end
858743
def to_postfix(tokens):
    
    precedence = {
        '/': 4,
        '*': 4,
        '+': 3,
        '-': 3,
        '^': 2,
        '(': 1
    }

    postfix = []
    opstack = []

    for token in tokens:
        if is_int(token):
            postfix.append(int(token))
        elif is_float(token):
            postfix.append(float(token))
        elif token in mathwords.CONSTANTS:
            postfix.append(mathwords.CONSTANTS[token])
        elif is_unary(token):
            opstack.append(token)
        elif token == '(':
            opstack.append(token)
        elif token == ')':
            top_token = opstack.pop()
            while top_token != '(':
                postfix.append(top_token)
                top_token = opstack.pop()
        else:
            while (opstack != []) and (
                precedence[opstack[-1]] >= precedence[token]
            ):
                postfix.append(opstack.pop())
            opstack.append(token)

    while opstack != []:
        postfix.append(opstack.pop())

    return postfix
625771
def Convert(self, metadata, value, token=None):
    """"""

    for r in self.BatchConvert([(metadata, value)], token=token):
      yield r
329204
def convert_ref3d(token, _class)
    # Split the ref at the ! symbol
    ext_ref, cell = token.split('!')

    # Convert the external reference part
    ext_ref = pack_ext_ref(ext_ref)

    # Convert the cell reference part
    row, col = cell_to_packed_rowcol(cell)

    # The ptg value depends on the class of the ptg.
    if    (_class == 0)
      ptgref = [@ptg['ptgRef3d']].pack("C")
    elsif (_class == 1)
      ptgref = [@ptg['ptgRef3dV']].pack("C")
    elsif (_class == 2)
      ptgref = [@ptg['ptgRef3dA']].pack("C")
    else
      exit "Unknown function class in formula\n"
    end

    ptgref + ext_ref + row + col
  end
1159963
function toOptionToken(token) {
    if (!/^-/.test(token)) {
        token = token.split(' ')[0];
        if (!/^(\[|<)/.test(token))
            token = "[" + token + "]";
    }
    var pre = token;
    var suffix = '';
    var reqIdx = token.indexOf('<');
    var optIdx = token.indexOf('[');
    var idx = !!~reqIdx ? reqIdx : !!~optIdx ? optIdx : null;
    if (idx) {
        pre = token.slice(0, idx);
        suffix = token.slice(idx);
    }
    pre = chek_1.split(pre.trim(), constants_2.SPLIT_CHARS).join('.').replace(/\s/g, '');
    return pre + ' ' + suffix;
}
532194
def convert(document):
    """"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
754684
def convert_range3d(token, _class)
    # Split the ref at the ! symbol
    ext_ref, range = token.split('!')

    # Convert the external reference part
    ext_ref = pack_ext_ref(ext_ref)

    # Split the range into 2 cell refs
    cell1, cell2 = range.split(':')

    # A range such as A:D is equivalent to A1:D65536, so add rows as required
    cell1 += '1'     unless cell1 =~ /\d/
    cell2 += '65536' unless cell2 =~ /\d/

    # Convert the cell references
    row1, col1 = cell_to_packed_rowcol(cell1)
    row2, col2 = cell_to_packed_rowcol(cell2)

    # The ptg value depends on the class of the ptg.
    if    (_class == 0)
      ptgarea = [@ptg['ptgArea3d']].pack("C")
    elsif (_class == 1)
      ptgarea = [@ptg['ptgArea3dV']].pack("C")
    elsif (_class == 2)
      ptgarea = [@ptg['ptgArea3dA']].pack("C")
    else
      exit "Unknown function class in formula\n"
    end

    ptgarea + ext_ref + row1 + row2 + col1+ col2
  end
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
83833
def _normalize(self, tokens):
        """"""
        if self.case_sensitive:
            return ' '.join(self.lexicon[t].normalized for t in tokens)
        else:
            return ' '.join(self.lexicon[t].lower for t in tokens)
150051
def pcfgtokensofw(self, word):
        
        tok = helper.tokens(word)

        sym = ['__{0}{1}__'.format(helper.whatchar(w), len(w))
               for w in tok]
        S = ['__S__' + ''.join(sym).replace('_', '') + '__']
        return S + sym + tok
1203409
def _convert_pflags(self, pflags):
        """"""
        if (pflags & SFTP_FLAG_READ) and (pflags & SFTP_FLAG_WRITE):
            flags = os.O_RDWR
        elif pflags & SFTP_FLAG_WRITE:
            flags = os.O_WRONLY
        else:
            flags = os.O_RDONLY
        if pflags & SFTP_FLAG_APPEND:
            flags |= os.O_APPEND
        if pflags & SFTP_FLAG_CREATE:
            flags |= os.O_CREAT
        if pflags & SFTP_FLAG_TRUNC:
            flags |= os.O_TRUNC
        if pflags & SFTP_FLAG_EXCL:
            flags |= os.O_EXCL
        return flags
544560
private void convertTokenDefinitions() throws GrammarException,
	    TreeException {
	tokenVisibility.clear();
	Map<String, ParseTreeNode> helpers = getHelperTokens();
	Map<String, ParseTreeNode> tokens = getTokens();
	convertTokenDefinitions(helpers, tokens);
    }
25308
def tokens(self):
        """"""
        spans = self.word_tokenizer.span_tokenize(self.text)
        toks = [Token(
            text=self.text[span[0]:span[1]],
            start=span[0] + self.start,
            end=span[1] + self.start,
            lexicon=self.lexicon
        ) for span in spans]
        return toks
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
464289
def process_tokens(
    tokens: Sequence[Token], *, end: str = "\n", sep: str = " "
) -> Tuple[str, str]:
    
    # Flatten the list of tokens in case some of them are of
    # class UnicodeSequence:
    flat_tokens = list()  # type: List[Token]
    for token in tokens:
        if isinstance(token, UnicodeSequence):
            flat_tokens.extend(token.tuple())
        else:
            flat_tokens.append(token)

    with_color = _process_tokens(flat_tokens, end=end, sep=sep, color=True)
    without_color = _process_tokens(flat_tokens, end=end, sep=sep, color=False)
    return (with_color, without_color)
621485
def tokens(self, instance):
        
        if not instance.pk:
            # e.g.: Use will create a new TOTP entry
            return "-"

        totp = TOTP(instance.bin_key, instance.step, instance.t0, instance.digits)

        tokens = []
        for offset in range(-instance.tolerance, instance.tolerance + 1):
            totp.drift = instance.drift + offset
            tokens.append(totp.token())

        return " ".join(["%s" % token for token in tokens])
1205305
def split_token_in_parts(token):
    
    result = []
    current = []
    for part in token + (':', ):
        if part == ':':
            if current:
                result.append(tuple(current))
                current = []
        else:
            current.append(part)

    return result