Handle tokens inside the list or outside the list.
165856
def __default(self, ast_token):
        """"""
        if self.list_level == 1:
            if self.list_entry is None:
                self.list_entry = ast_token
            elif not isinstance(ast_token, type(self.list_entry)):
                self.final_ast_tokens.append(ast_token)
        elif self.list_level == 0:
            self.final_ast_tokens.append(ast_token)
222822
public void process(Iterator<String> text) {
        String nextToken = null, curToken = null;
        // Base case for the next token buffer to ensure we always have two
        // valid tokens present
        if (text.hasNext())
            nextToken = text.next();
        while (text.hasNext()) {
            curToken = nextToken;
            nextToken = text.next();
            // Only process bigrams where the two tokens weren't excluded by the
            // token filter
            if (!(excludeToken(curToken) || excludeToken(nextToken)))
                processBigram(curToken, nextToken);
        }
    }
509172
def __parse(self, tokenlist):
        """"""
        #empty stack
        #iterate over symbollist
        tokenlist = [x for x in tokenlist]
        if not isinstance(tokenlist, list):
            raise TypeError("Expected list, got %s" % tokenlist.__class__.__name__)
        LOG.debug("get_trees: checking list: " + str(tokenlist))
        stack = [(0, Extended_S)]
        while True:
            state = stack[-1][0]
            if len(tokenlist):#FIXME: tokenlist with one element is reported as false
                token = tokenlist[0]
            else:
                token = EndSymbol()
            newdic = self.__parsertable.insert(state, token)
            action = newdic["action"]
            if action == "Fail":
                return False
            elif action == "Accept":
                return True
            if action == "Reduce":
                reductionrule = newdic["rule"]
                #TODO extract len(right side) of the rule and insert left side
                for rsymbol in reversed(reductionrule.rightside):
                    state, symbol = stack.pop() # TODO: check
                state = stack[-1][0]
                state = self.__parsertable.goto(state,reductionrule.leftside[0])
                stack.append((state, reductionrule.leftside[0]))
            elif action == "Shift":
                stack.append((newdic['dest'], tokenlist.pop(0)))
            else:
                raise ValueError("Unknown action")
        return False
134586
def tokenize_list(self, text):
        
        return [self.get_record_token(record) for record in self.analyze(text)]
792189
protected void processElement(List<PatternToken> patternTokens) {
    int counter = 0;
    for (PatternToken pToken : patternTokens) {
        if (pToken.getPhraseName() != null && counter > 0 && pToken.isReferenceElement()) {
            int tokRef = pToken.getMatch().getTokenRef();
            pToken.getMatch().setTokenRef(tokRef + counter - 1);
            String offsetToken = pToken.getString().replace("\\" + tokRef,
                    "\\" + (tokRef + counter - 1));
            pToken.setStringElement(offsetToken);
        }
      counter++;
    }
  }
764704
def as_tokens(ctx: List[ParserRuleContext]) -> List[str]:
    
    return [as_token(e) for e in ctx]
179539
def none_coalesce_handle(tokens):
    """"""
    if len(tokens) == 1:
        return tokens[0]
    elif tokens[0].isalnum():
        return "({b} if {a} is None else {a})".format(
            a=tokens[0],
            b=none_coalesce_handle(tokens[1:]),
        )
    else:
        return "(lambda {x}: {b} if {x} is None else {x})({a})".format(
            x=none_coalesce_var,
            a=tokens[0],
            b=none_coalesce_handle(tokens[1:]),
        )
682209
def build(self, tokenlist):
        """"""
        self._tokens = tokenlist
        self._tokens.reverse()
        self._push()
        while self._tokens:
            node = self._handle_token(self._tokens.pop())
            self._write(node)
        return self._pop()
510291
def handle_term(self, _, __, tokens: ParseResults) -> ParseResults:
        """"""
        self.ensure_node(tokens)
        return tokens
158626
def handle_p_s_r(tokens, options)
      new_tokens = [tokens[1], tokens[2], tokens[0]]
      handle_s_r_p(new_tokens, options)
    end
201974
def handle_s_r_p(tokens, options)
      span = Span.new(self.now, self.now + 1)

      handle_srp(tokens, span, options)
    end
496079
def tokenize(self, token_list):
    """"""
    return [
        self._vocab_dict.get(token, self._vocab_dict[self.UNK])
        for token in token_list
    ]
1284362
function handleExprList (node, options) {
  const values = node.value.map((valueNode) => traverse(valueNode, options)).join(',')
  return `(${values})`
}
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
1351593
protected void parseElse() {
        Node parent = peek();
        if (!(parent instanceof IfNode)) {
            return;
        }
        IfNode ifNode = (IfNode) pop();
        ElseNode elseNode = new ElseNode();
        ifNode.setElseNode(elseNode);
        push(elseNode);
        tokenizer.skipWhitespace();
    }
1118763
def decorator_handle(tokens):
    """"""
    defs = []
    decorates = []
    for i, tok in enumerate(tokens):
        if "simple" in tok and len(tok) == 1:
            decorates.append("@" + tok[0])
        elif "test" in tok and len(tok) == 1:
            varname = decorator_var + "_" + str(i)
            defs.append(varname + " = " + tok[0])
            decorates.append("@" + varname)
        else:
            raise CoconutInternalException("invalid decorator tokens", tok)
    return "\n".join(defs + decorates) + "\n"
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens