Parses a code block with no parentheses and no commas.  After it is done there should be a single token left,
which is returned.
169523
protected TokenList.Token parseBlockNoParentheses(TokenList tokens, Sequence sequence, boolean insideMatrixConstructor) {

        // search for matrix bracket operations
        if( !insideMatrixConstructor ) {
            parseBracketCreateMatrix(tokens, sequence);
        }

        // First create sequences from anything involving a colon
        parseSequencesWithColons(tokens, sequence );

        // process operators depending on their priority
        parseNegOp(tokens, sequence);
        parseOperationsL(tokens, sequence);
        parseOperationsLR(new Symbol[]{Symbol.POWER, Symbol.ELEMENT_POWER}, tokens, sequence);
        parseOperationsLR(new Symbol[]{Symbol.TIMES, Symbol.RDIVIDE, Symbol.LDIVIDE, Symbol.ELEMENT_TIMES, Symbol.ELEMENT_DIVIDE}, tokens, sequence);
        parseOperationsLR(new Symbol[]{Symbol.PLUS, Symbol.MINUS}, tokens, sequence);

        // Commas are used in integer sequences.  Can be used to force to compiler to treat - as negative not
        // minus.  They can now be removed since they have served their purpose
        stripCommas(tokens);

        // now construct rest of the lists and combine them together
        parseIntegerLists(tokens);
        parseCombineIntegerLists(tokens);

        if( !insideMatrixConstructor ) {
            if (tokens.size() > 1) {
                System.err.println("Remaining tokens: "+tokens.size);
                TokenList.Token t = tokens.first;
                while( t != null ) {
                    System.err.println("  "+t);
                    t = t.next;
                }
                throw new RuntimeException("BUG in parser.  There should only be a single token left");
            }
            return tokens.first;
        } else {
            return null;
        }
    }
802640
function getBlockquoteToken(token) {
  token.content = joinArrString(parseBlockquote(token));
  token.length = getTokensLength(token.content);
  delete token.markup;
  return token;
}
548952
public function parseNextBlock()
    {
        // Init block variables.
        $found_var = false;
        $found_equal = false;
        $found_semicolon = false;
        $found_open_tag = false;
        $found_close_tag = false;
        $end_block = false;
        $var_name = '';
        $var_value = '';
        $line = 0;

        // Loop through tokens until end of block or file.
        while (!$end_block && $this->tokens->valid()) {
            // Init loop variables
            $add_name = false;

            // Get normed token array
            $token = static::normalizeToken($this->tokens->current());
            $line = max($token[self::T_LINE], $line);
            $this->logger->debug('Found token ' . static::getTokenName($token) . " at line $line.");

            // Closing block allow only one whitespace.
            if ($found_semicolon  || $found_open_tag || $found_close_tag) {
                if ($token[self::T_KEY] == T_WHITESPACE) {
                    $end_block = true;
                    if (!$this->test_mode && $token[self::T_VALUE] != "\n") {
                        // Set only one line return after a semicolon or close tag.
                        $this->logger->notice("Removing spaces at line $line.");
                        $token[self::T_VALUE] = "\n";
                    }
                } else {
                    // Stop processing now.
                    break;
                }
            } else {
                switch ($token[self::T_KEY]) {
                    // Comments and whitespace are allowed anywhere.
                    case T_COMMENT:
                    case T_DOC_COMMENT:
                    case T_WHITESPACE:
                        break;
                    case T_OPEN_TAG:
                        $found_open_tag = true;
                        break;
                    // New variable
                    case T_VARIABLE:
                        if (!$found_var) {
                            // Set that we found a var definition and record the name
                            $found_var = true;
                            $add_name = true;
                        } else {
                            // We shouldn't see a variable definition.
                            throw new \Exception('Unexpected T_VARIABLE ' . $token[self::T_VALUE] . " at line $line.");
                        }
                        break;
                    // Variable name..
                    case '[':
                    case ']':
                    case T_CONSTANT_ENCAPSED_STRING:
                        if ($found_var) {
                            if (!$found_equal) {
                                // If inside variable definition, add name for sorting.
                                $add_name = true;
                            }
                        } else {
                            // Not in variable definition or value.
                            $name = static::getTokenName($token);
                            throw new \Exception('Unexpected ' . $name . " at line $line.");
                        }
                        break;
                    case '=':
                        // Equal allowed only after var definition
                        if ($found_var && !$found_equal) {
                            $found_equal = true;
                        } else {
                            throw new \Exception("Unexpected = at line $line.");
                        }
                        break;
                    case ';':
                        if ($found_var && !$found_equal) {
                            throw new \Exception("Unexpected ; at line $line.");
                        }
                        $found_semicolon = true;
                        break;
                    case T_CLOSE_TAG:
                        if ($found_var) {
                            throw new \Exception("Unexpected T_CLOSE_TAG at line $line.");
                        }
                        $found_close_tag = true;
                        break;
                    default:
                        if (!$found_var || !$found_equal) {
                            $name = static::getTokenName($token);
                            throw new \Exception('Unexpected ' . $name . " at line $line.");
                        }
                        break;
                }
            }
            if ($add_name) {
                $var_name .= $token[self::T_VALUE];
            }
            $var_value .= $token[self::T_VALUE];
            $this->logger->debug('Advance to next token.');
            $this->tokens->next();
        }
        if ($found_var && !$found_semicolon) {
            throw new \Exception('Missing ; before end of file.');
        }
        if (!$this->test_mode && "\n" != substr($var_value, -1)) {
            $this->logger->info('Add return line to end of block.');
            $var_value .= "\n";
        }
        if ($found_close_tag) {
            $this->end_blocks[] = $var_value;
        } elseif (!empty($var_name)) {
            if ($this->test_mode) {
                // Try to recreate the file as is.
                $this->var_blocks[] = $var_value;
            } else {
                // Strip $GLOBALS from variables names for sorting.
                // This will merge local and global version of the same var name.
                $var_name = $this->normalizeVariableName($var_name);
                // Warnings if key already exists.
                $this->checkVarName($var_name);
                $this->var_blocks[$var_name] = $var_value;
            }
        } else {
            $this->empty_blocks[] = $var_value;
        }
    }
1286359
def memoized_parse_block(code):
    """"""
    success, result = parse_block_memo.get(code, (None, None))
    if success is None:
        try:
            parsed = COMPILER.parse_block(code)
        except Exception as err:
            success, result = False, err
        else:
            success, result = True, parsed
        parse_block_memo[code] = (success, result)
    if success:
        return result
    else:
        raise result
937576
def parse(data)
      sections = []
      docs, code = [], []
      lines = data.split("\n")

      # Skip shebang and encoding information
      lines.shift if lines[0] =~ /^\#\!/
      lines.shift if lines[0] =~ /coding[:=]\s*[-\w.]+/

      in_comment_block = false
      lines.each do |line|
        # If we're currently in a comment block, check whether the line matches
        # the _end_ of a comment block
        if in_comment_block
          if line.match(@@block_comment_end)
            in_comment_block = false
          else
            docs << line
          end
        # Otherwise, check whether the line matches the beginning of a block, or
        # a single-line comment all on it's lonesome. In either case, if there's
        # code, start a new section
        else
          if line.match(@@block_comment_start)
            in_comment_block = true
            if code.any?
              sections << [docs, code]
              docs, code = [], []
            end
          elsif line.match(@@comment_pattern)
            if code.any?
              sections << [docs, code]
              docs, code = [], []
            end
            docs << line.sub(@@comment_pattern, '')
          else
            code << line
          end
        end
      end

      sections << [docs, code] if docs.any? || code.any?
      normalize_leading_spaces(sections)
    end
803830
def _parse_block_header(line):
    
    level = line[1]
    one, _rest = line[4:].split(')=', 1)
    name, _rest = _rest.split('(', 1)
    seqcount, _rest = _rest.split(')', 1)
    params = _rest.strip('{}:')
    # try:
    #     params = dict((key, float(val))
    #             for key, val in (pair.split('=')
    #                 for pair in _rest[1:-2].split(',')))
    # except ValueError:
    #     # Couldn't convert params to key-val pairs, for whatever reason
    #     logging.warn("Failed to parse CMA params: %s", _rest[1:-2])
    #     params = {}
    return int(level), int(one), name, int(seqcount), params
662371
private ExprKraken parseCmpExpr()
  {
    // Expr left = parseConcatExpr();
    ExprKraken left = parseAddExpr();

    Token token = scanToken();
    
    boolean isNot = false;
    
    /*

    if (token == NOT) {
      isNot = true;

      token = scanToken();

      if (token != BETWEEN && token != LIKE && token != IN) {
        _token = token;

        return left;
      }
    }
    */

    switch (token) {
    case EQ:
      return new BinaryExpr(BinaryOp.EQ, left, parseAddExpr());
      
    case NE:
      return new BinaryExpr(BinaryOp.NE, left, parseAddExpr());
      
    case LT:
      return new BinaryExpr(BinaryOp.LT, left, parseAddExpr());
      
    case LE:
      return new BinaryExpr(BinaryOp.LE, left, parseAddExpr());
      
    case GT:
      return new BinaryExpr(BinaryOp.GT, left, parseAddExpr());
      
    case GE:
      return new BinaryExpr(BinaryOp.GE, left, parseAddExpr());

    case BETWEEN:
      {
        ExprKraken min = parseAddExpr();

        token = scanToken();
        if (token != Token.AND)
          throw error(L.l("expected AND at '{0}'", token));

        ExprKraken max = parseAddExpr();

        return new BetweenExpr(left, min, max, isNot);
      }
/*
    case LT:
    case LE:
    case GT:
    case GE:
    case NE:
      return new CmpExpr(left, parseConcatExpr(), token);

    case BETWEEN:
      {
        Expr min = parseConcatExpr();

        token = scanToken();
        if (token != AND)
          throw error(L.l("expected AND at '{0}'", tokenName(token)));

        Expr max = parseConcatExpr();

        return new BetweenExpr(left, min, max, isNot);
      }

    case IS:
      {
        token = scanToken();
        isNot = false;
        if (token == NOT) {
          token = scanToken();
          isNot = true;
        }

        if (token == NULL)
          return new IsNullExpr(left, isNot);
        else
          throw error(L.l("expected NULL at '{0}'", tokenName(token)));
      }

    case LIKE:
      {
        token = scanToken();

        if (token == STRING)
          return new LikeExpr(left, _lexeme, isNot);
        else
          throw error(L.l("expected string at '{0}'", tokenName(token)));
      }

    case IN:
      {
        HashSet<String> values = parseInValues();

        return new InExpr(left, values, isNot);
      }
*/
    default:
      _token = token;
      return left;
    }
  }
449107
public final void block() throws RecognitionException {
		int block_StartIndex = input.index();


		            increaseLevel();
		        
		try {
			if ( state.backtracking>0 && alreadyParsedRule(input, 80) ) { return; }

			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:693:5: ( '{' ( blockStatement )* '}' )
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:693:7: '{' ( blockStatement )* '}'
			{
			match(input,121,FOLLOW_121_in_block2769); if (state.failed) return;
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:693:11: ( blockStatement )*
			loop104:
			while (true) {
				int alt104=2;
				int LA104_0 = input.LA(1);
				if ( ((LA104_0 >= CharacterLiteral && LA104_0 <= DecimalLiteral)||LA104_0==ENUM||LA104_0==FloatingPointLiteral||(LA104_0 >= HexLiteral && LA104_0 <= Identifier)||(LA104_0 >= OctalLiteral && LA104_0 <= StringLiteral)||LA104_0==29||LA104_0==36||(LA104_0 >= 40 && LA104_0 <= 41)||(LA104_0 >= 44 && LA104_0 <= 45)||(LA104_0 >= 52 && LA104_0 <= 53)||LA104_0==58||(LA104_0 >= 63 && LA104_0 <= 67)||(LA104_0 >= 70 && LA104_0 <= 73)||(LA104_0 >= 75 && LA104_0 <= 77)||(LA104_0 >= 79 && LA104_0 <= 80)||(LA104_0 >= 82 && LA104_0 <= 83)||(LA104_0 >= 85 && LA104_0 <= 87)||LA104_0==90||(LA104_0 >= 92 && LA104_0 <= 98)||(LA104_0 >= 100 && LA104_0 <= 112)||(LA104_0 >= 114 && LA104_0 <= 121)||LA104_0==126) ) {
					alt104=1;
				}

				switch (alt104) {
				case 1 :
					// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:693:11: blockStatement
					{
					pushFollow(FOLLOW_blockStatement_in_block2771);
					blockStatement();
					state._fsp--;
					if (state.failed) return;
					}
					break;

				default :
					break loop104;
				}
			}

			match(input,125,FOLLOW_125_in_block2774); if (state.failed) return;
			}

			if ( state.backtracking==0 ) {
			            if( localVariableLevel <= 1 ) {
			                // this is the top level block, so set the top level declarations
			                rootBlockDescr.setInScopeLocalVars( getLocalDeclarations() );
			            }
			            decreaseLevel();
			        }
		}
		catch (RecognitionException re) {
			reportError(re);
			recover(input,re);
		}
		finally {
			// do for sure before leaving
			if ( state.backtracking>0 ) { memoize(input, 80, block_StartIndex); }

		}
	}
1175221
function(code, ctx){
    // Wrap code blocks with {}.
    // we only wrap unbuffered code blocks ATM
    // since they are usually flow control
    // Buffer code
    //
    var ast = [];
    if (code.buffer) {
      var val = code.val.trim();
      var tpl = tpl_interp;
      var tplv = {
        VALUE: code.astVal || this.parseExpr(code.val)
      }
      if (code.mustEscape !== false) {
        tpl = tpl_interp_escape;
        tplv.ESCAPE = this.runtime('escape', true);
      }
      push.apply(ast, this.bufferAST(tpl(tplv).expression));
    } else {

      var val = code.val.trim();
      this.codeBuffer += "\n" + val;

      if (code.block) {
        this.codeIndex++;
        var marker = "PUGMARKER"+this.codeIndex;
        this.codeBuffer += "\n{" + marker + "}\n";
        // snaphsot current unbuffered code level
        // this is necessary to accept embedded code blocks
        // - if (true) {
        //   - var a = 1;
        // - }
        // but breaks the "should be reasonably fast" compile test
        // note: we should add a test for this in pug
        var savedCodeBuffer = this.codeBuffer;
        var savedCodeMarker = this.codeMarker;
        var savedCodeIndex = this.codeIndex;

        this.codeBuffer = '_=function*(){';
        this.codeMarker = {};
        this.codeIndex = -1;

        var body = this.visit(code.block, code);

        this.codeBuffer = savedCodeBuffer;
        this.codeMarker = savedCodeMarker;
        this.codeIndex = savedCodeIndex;
       
        this.codeMarker[marker] = body; 
      }

      var idx = ctx.nodes.indexOf(code) + 1;
      if (idx == ctx.nodes.length || ctx.nodes[idx].type != 'Code' || ctx.nodes[idx].buffer) {
        try {
          var src = this.codeBuffer + '}';
          var tpl = babelTemplate(src);
          push.apply(ast, tpl(this.codeMarker).expression.right.body.body);
          this.codeBuffer = '_=function*(){';
          this.codeIndex = -1;
          this.codeMarker = {};
        } catch(e) {
          var codeError = this.codeBuffer.substr(14).trim()
          this.error('Unbuffered code structure could not be parsed; ' + e.message + ' in ' + codeError, codeError, code)
        }
      }

    }
    return ast;
  }
426908
function parseShortcode(innerShortcode) {
  var trimmedInnerShortcode = innerShortcode.trim();

  // If no shortcode, it was blank between the blocks - return nothing.
  if (!trimmedInnerShortcode) return;

  // If no whitespace, then shortcode is just name with no attributes.
  if (!hasWhiteSpace(trimmedInnerShortcode)) {
    return { identifier: trimmedInnerShortcode, attributes: {} };
  }

  var splitShortcode = trimmedInnerShortcode.match(/^(\S+)\s(.*)/).slice(1);
  var shortcodeName = splitShortcode[0];
  var attributeString = splitShortcode[1];
  var attributes = parseShortcodeAttributes(attributeString);

  // If no attributes parsed, something went wrong - return nothing.
  if (!attributes) return;

  return {
    identifier: shortcodeName,
    attributes: attributes
  };
}
1351042
def token_at_cursor(code, pos=0):
    
    l = len(code)
    end = start = pos
    # Go forwards while we get alphanumeric chars
    while end < l and code[end].isalpha():
        end += 1
    # Go backwards while we get alphanumeric chars
    while start > 0 and code[start-1].isalpha():
        start -= 1
    # If previous character is a %, add it (potential magic)
    if start > 0 and code[start-1] == '%':
        start -= 1
    return code[start:end], start
509172
def __parse(self, tokenlist):
        """"""
        #empty stack
        #iterate over symbollist
        tokenlist = [x for x in tokenlist]
        if not isinstance(tokenlist, list):
            raise TypeError("Expected list, got %s" % tokenlist.__class__.__name__)
        LOG.debug("get_trees: checking list: " + str(tokenlist))
        stack = [(0, Extended_S)]
        while True:
            state = stack[-1][0]
            if len(tokenlist):#FIXME: tokenlist with one element is reported as false
                token = tokenlist[0]
            else:
                token = EndSymbol()
            newdic = self.__parsertable.insert(state, token)
            action = newdic["action"]
            if action == "Fail":
                return False
            elif action == "Accept":
                return True
            if action == "Reduce":
                reductionrule = newdic["rule"]
                #TODO extract len(right side) of the rule and insert left side
                for rsymbol in reversed(reductionrule.rightside):
                    state, symbol = stack.pop() # TODO: check
                state = stack[-1][0]
                state = self.__parsertable.goto(state,reductionrule.leftside[0])
                stack.append((state, reductionrule.leftside[0]))
            elif action == "Shift":
                stack.append((newdic['dest'], tokenlist.pop(0)))
            else:
                raise ValueError("Unknown action")
        return False
483128
function parse(tokens) {
    var token;
    return process();

    function process(operation) {
        operation = operation || null;
        var args = [];

        while (next()) {
            if (token.type == 'boundary') {
                if (token.value == '(') args.push(process());
                else if (token.value == ')') return node(operation, args);
            }

            else if (token.type == 'variable') {
                args.push(node('substitution', [ token.value ]));
                if (isUnary(operation)) return node(operation, args);
            }

            else if (token.type == 'operator') {
                if (isUnary(token.value)) {
                    args.push(process(token.value));
                    continue;
                }

                if (operation) {
                    var tmp = args.slice(0);
                    args = [];
                    args.push(node(operation, tmp));
                }

                operation = token.value;
            }
        }

        return node(operation, args);
    }

    function next() {
        //TODO: use pointer instead of shifting
        //(parse would not need to clone tokens array)
        return (token = tokens.shift());
    }

    function node(action, args) {
        return {
            action: translate(action),
            args: args
        };
    }

    function translate(operator) {
        var map = {
            '!': 'negation',
            '|': 'disjunction',
            '&': 'conjunction',
            '->': 'implication',
            '<->': 'equivalence'
        };

        return map[operator] || operator;
    }

    function isUnary(op) {
        return op === '!';
    }

    function syntaxError() {
        throw new Error('Syntax error!');
    }
}
1302431
def parse(self, text):
        
        #: Do not process empty strings (Issue #3)
        if text.strip() == "":
            return ""
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif text.strip() in PUNCTUATION:
            _sym = text.strip()
            if _sym in tuple('.?!'):
                _tag = "."
            else:
                _tag = _sym
            if self.lemmata:
                return "{0}/{1}/O/O/{0}".format(_sym, _tag)
            else:
                return "{0}/{1}/O/O".format(_sym, _tag)
        if self.tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(text))
        else:
            _tokenized = text

        _parsed = pattern_parse(_tokenized,
                                # text is tokenized before it is passed on to
                                # pattern.de.parse
                                tokenize=False,
                                tags=self.tags, chunks=self.chunks,
                                relations=self.relations, lemmata=self.lemmata,
                                encoding=self.encoding, tagset=self.tagset)
        if self.pprint:
            _parsed = pattern_pprint(_parsed)

        return _parsed
1125074
private Token attributesBlock() {
        Matcher matcher = scanner.getMatcherForPattern("^&attributes\\b");
        if (matcher.find(0) && matcher.group(0) != null) {
            this.scanner.consume(11);
            CharacterParser.Match match = this.bracketExpression();
            this.scanner.consume(match.getEnd()+1);
            return new AttributesBlock(match.getSrc(),lineno);
        }
        return null;
    }
296831
def parse_expr(tokens, options):
    """"""
    seq = parse_seq(tokens, options)
    if tokens.current() != '|':
        return seq
    result = [Required(*seq)] if len(seq) > 1 else seq
    while tokens.current() == '|':
        tokens.move()
        seq = parse_seq(tokens, options)
        result += [Required(*seq)] if len(seq) > 1 else seq
    return [Either(*result)] if len(result) > 1 else result
344725
def get_codeblock(language, text):
    
    rst = "\n\n.. code-block:: " + language + "\n\n"
    for line in text.splitlines():
        rst += "\t" + line + "\n"

    rst += "\n"
    return rst