Searches for pairs of parentheses and processes blocks inside of them.  Embedded parentheses are handled
with no problem.  On output only a single token should be in tokens.
@param tokens List of parsed tokens
@param sequence Sequence of operators
763796
protected void handleParentheses( TokenList tokens, Sequence sequence ) {
        // have a list to handle embedded parentheses, e.g. (((((a)))))
        List<TokenList.Token> left = new ArrayList<TokenList.Token>();

        // find all of them
        TokenList.Token t = tokens.first;
        while( t != null ) {
            TokenList.Token next = t.next;
            if( t.getType() == Type.SYMBOL ) {
                if( t.getSymbol() == Symbol.PAREN_LEFT  )
                    left.add(t);
                else if( t.getSymbol() == Symbol.PAREN_RIGHT ) {
                    if( left.isEmpty() )
                        throw new ParseError(") found with no matching (");

                    TokenList.Token a = left.remove(left.size()-1);

                    // remember the element before so the new one can be inserted afterwards
                    TokenList.Token before = a.previous;

                    TokenList sublist = tokens.extractSubList(a,t);
                    // remove parentheses
                    sublist.remove(sublist.first);
                    sublist.remove(sublist.last);

                    // if its a function before () then the () indicates its an input to a function
                    if( before != null && before.getType() == Type.FUNCTION ) {
                        List<TokenList.Token> inputs = parseParameterCommaBlock(sublist, sequence);
                        if (inputs.isEmpty())
                            throw new ParseError("Empty function input parameters");
                        else {
                            createFunction(before, inputs, tokens, sequence);
                        }
                    } else if( before != null && before.getType() == Type.VARIABLE &&
                            before.getVariable().getType() == VariableType.MATRIX ) {
                        // if it's a variable then that says it's a sub-matrix
                        TokenList.Token extract = parseSubmatrixToExtract(before,sublist, sequence);
                        // put in the extract operation
                        tokens.insert(before,extract);
                        tokens.remove(before);
                    } else {
                        // if null then it was empty inside
                        TokenList.Token output = parseBlockNoParentheses(sublist,sequence, false);
                        if (output != null)
                            tokens.insert(before, output);
                    }
                }
            }
            t = next;
        }

        if( !left.isEmpty())
            throw new ParseError("Dangling ( parentheses");
    }
359816
public void setSyntaxTokens(java.util.Collection<SyntaxToken> syntaxTokens) {
        if (syntaxTokens == null) {
            this.syntaxTokens = null;
            return;
        }

        this.syntaxTokens = new java.util.ArrayList<SyntaxToken>(syntaxTokens);
    }
766981
private String getPatternCode(final List<Token> tokens) {
        final StringBuilder sb = new StringBuilder();
        sb.append(tokens.size());
        for (final Token token : tokens) {
            sb.append(token.getType().ordinal());
        }
        return sb.toString();
    }
600006
public void setImportSequences(String[] sequences) {
        for (String s : sequences) {
            s = s.trim();
            if (s.length() > 0) {
                String[] ts = s.split("\\s+");
                List<String> sequence = new ArrayList<String>();
                for (String t : ts) {
                    t = t.trim();
                    if (t.length() > 0) {
                        sequence.add(t);
                    }
                }
                this.importSequences.add(new StringSequence(sequence));
            }
        }
    }
363412
protected TokenList.Token parseSubmatrixToExtract(TokenList.Token variableTarget,
                                                      TokenList tokens, Sequence sequence) {


        List<TokenList.Token> inputs = parseParameterCommaBlock(tokens, sequence);

        List<Variable> variables = new ArrayList<Variable>();

        // for the operation, the first variable must be the matrix which is being manipulated
        variables.add(variableTarget.getVariable());

        addSubMatrixVariables(inputs, variables);
        if( variables.size() != 2 && variables.size() != 3 ) {
            throw new ParseError("Unexpected number of variables.  1 or 2 expected");
        }

        // first parameter is the matrix it will be extracted from.  rest specify range
        Operation.Info info;

        // only one variable means its referencing elements
        // two variables means its referencing a sub matrix
        if( inputs.size() == 1 ) {
            Variable varA = variables.get(1);
            if( varA.getType() == VariableType.SCALAR ) {
                info = functions.create("extractScalar", variables);
            } else {
                info = functions.create("extract", variables);
            }
        } else if( inputs.size() == 2 ) {
            Variable varA = variables.get(1);
            Variable varB = variables.get(2);

            if( varA.getType() == VariableType.SCALAR && varB.getType() == VariableType.SCALAR) {
                info = functions.create("extractScalar", variables);
            } else {
                info = functions.create("extract", variables);
            }
        } else {
            throw new ParseError("Expected 2 inputs to sub-matrix");
        }

        sequence.addOperation(info.op);

        return new TokenList.Token(info.output);
    }
303770
private void checkInvariants() {
        for (int i = 0; i < tokens.size(); i++) {
            Token token = tokens.get(i);

            Preconditions.checkArgument(token.getID() == i + 1,
                    String.format("Tokens should be numbered consecutively, starting at 1: %s", token));

            checkHead(token, token.getHead());
            checkHead(token, token.getPHead());
        }
    }
42012
protected void parseBracketCreateMatrix(TokenList tokens, Sequence sequence) {
        List<TokenList.Token> left = new ArrayList<TokenList.Token>();

        TokenList.Token t = tokens.getFirst();

        while( t != null ) {
            TokenList.Token next = t.next;
            if( t.getSymbol() == Symbol.BRACKET_LEFT ) {
                left.add(t);
            } else if( t.getSymbol() == Symbol.BRACKET_RIGHT ) {
                if( left.isEmpty() )
                    throw new RuntimeException("No matching left bracket for right");

                TokenList.Token start = left.remove(left.size() - 1);

                // Compute everything inside the [ ], this will leave a
                // series of variables and semi-colons hopefully
                TokenList bracketLet = tokens.extractSubList(start.next,t.previous);
                parseBlockNoParentheses(bracketLet, sequence, true);
                MatrixConstructor constructor = constructMatrix(bracketLet);

                // define the matrix op and inject into token list
                Operation.Info info = Operation.matrixConstructor(constructor);
                sequence.addOperation(info.op);

                tokens.insert(start.previous, new TokenList.Token(info.output));

                // remove the brackets
                tokens.remove(start);
                tokens.remove(t);
            }

            t = next;
        }

        if( !left.isEmpty() )
            throw new RuntimeException("Dangling [");
    }
544778
public void skipUntil(final TokenType... tokenTypes) {
        Set<TokenType> tokenTypeSet = Sets.newHashSet(tokenTypes);
        tokenTypeSet.add(Assist.END);
        while (!tokenTypeSet.contains(lexer.getCurrentToken().getType())) {
            lexer.nextToken();
        }
    }
771318
public static <T> Parser<T> sequence(
      Parser<?> p1, Parser<?> p2, Parser<?> p3, Parser<T> p4) {
    return sequence(p1, p2, p3, p4, InternalFunctors.<Object, Object, Object, T>lastOfFour());
  }
1167657
public final List<SequenceLabel> getSequencesFromSpans(final String[] tokens,
      final Span[] seqSpans) {
    final List<SequenceLabel> sequences = new ArrayList<SequenceLabel>();
    for (final Span seqSpan : seqSpans) {
      final String seqString = seqSpan.getCoveredText(tokens);
      final String seqType = seqSpan.getType();
      final SequenceLabel sequence = this.sequenceFactory
          .createSequence(seqString, seqType, seqSpan);
      sequences.add(sequence);
    }
    return sequences;
  }
882574
private void process(Iterator<String> tokens) {
        // NOTE: this method is intentionally private to ensure that the
        // IteratorFactory.tokenize() tokenization scheme is enforced on the
        // input data
        long numTokens = 0;
        while (tokens.hasNext()) {
            String token = tokens.next();
            if (doLowerCasing)
                token = token.toLowerCase();
            if (token.matches("[0-9]+"))
                token = "<NUM>";
            if (token.matches("[^\\w\\s;:\\(\\)\\[\\]'!/&?\",\\.<>]"))
                continue;

            Integer count = tokenToCount.get(token);
            tokenToCount.put(token, (count == null) ? 1 : 1 + count);
            numTokens++;
            if (numTokens % UPDATE_INTERVAL == 0 
                    && LOGGER.isLoggable(Level.FINE))
                LOGGER.fine("Processed " + numTokens + " tokens.  Currently " 
                            + tokenToCount.size() + " unique tokens");
        }
    }
338069
public synchronized void tagNamedEntity(List<CoreLabel> tokens) {
        boolean isPOSTagged = tokens.parallelStream().filter(x -> x.tag() == null).count() == 0;
        if (!isPOSTagged) {
            throw new RuntimeException("Please Run POS Tagger before Named Entity tagger.");
        }
        if (ners != null) {
            try {
                ners.stream().forEach(ner -> ner.classify(tokens));
            } catch (Exception e) {
                /* edu.stanford.nlp.util.RuntimeInterruptedException: java.lang.InterruptedException
                 at edu.stanford.nlp.util.HashIndex.addToIndex(HashIndex.java:173) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher$BranchStates.newBid(SequenceMatcher.java:902) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher$MatchedStates.<init>(SequenceMatcher.java:1288) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher.getStartStates(SequenceMatcher.java:709) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher.findMatchStartBacktracking(SequenceMatcher.java:488) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher.findMatchStart(SequenceMatcher.java:449) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher.find(SequenceMatcher.java:341) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher.findNextNonOverlapping(SequenceMatcher.java:365) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ling.tokensregex.SequenceMatcher.find(SequenceMatcher.java:437) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.NumberNormalizer.findNumbers(NumberNormalizer.java:452) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.NumberNormalizer.findAndMergeNumbers(NumberNormalizer.java:721) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.time.TimeExpressionExtractorImpl.extractTimeExpressions(TimeExpressionExtractorImpl.java:184) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.time.TimeExpressionExtractorImpl.extractTimeExpressions(TimeExpressionExtractorImpl.java:178) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.time.TimeExpressionExtractorImpl.extractTimeExpressionCoreMaps(TimeExpressionExtractorImpl.java:116) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.time.TimeExpressionExtractorImpl.extractTimeExpressionCoreMaps(TimeExpressionExtractorImpl.java:104) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.regexp.NumberSequenceClassifier.runSUTime(NumberSequenceClassifier.java:340) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.regexp.NumberSequenceClassifier.classifyWithSUTime(NumberSequenceClassifier.java:138) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.regexp.NumberSequenceClassifier.classifyWithGlobalInformation(NumberSequenceClassifier.java:101) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.NERClassifierCombiner.recognizeNumberSequences(NERClassifierCombiner.java:267) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.NERClassifierCombiner.classifyWithGlobalInformation(NERClassifierCombiner.java:231) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 at edu.stanford.nlp.ie.NERClassifierCombiner.classify(NERClassifierCombiner.java:218) ~[stanford-corenlp-3.5.2.jar:3.5.2]
                 */
                LOG.warn("NER Classifier err for: " + tokens.stream().map(CoreLabel::word).collect(Collectors.joining(StringUtils.SPACE)));
            }
        }
    }
1110303
private List<Variable> parseAssignRange(Sequence sequence, TokenList tokens, TokenList.Token t0) {
        // find assignment symbol
        TokenList.Token tokenAssign = t0.next;
        while( tokenAssign != null && tokenAssign.symbol != Symbol.ASSIGN ) {
            tokenAssign = tokenAssign.next;
        }

        if( tokenAssign == null )
            throw new ParseError("Can't find assignment operator");

        // see if it is a sub matrix before
        if( tokenAssign.previous.symbol == Symbol.PAREN_RIGHT ) {
            TokenList.Token start = t0.next;
            if( start.symbol != Symbol.PAREN_LEFT )
                throw new ParseError(("Expected left param for assignment"));
            TokenList.Token end = tokenAssign.previous;
            TokenList subTokens = tokens.extractSubList(start,end);
            subTokens.remove(subTokens.getFirst());
            subTokens.remove(subTokens.getLast());

            handleParentheses(subTokens,sequence);

            List<TokenList.Token> inputs = parseParameterCommaBlock(subTokens, sequence);

            if (inputs.isEmpty())
                throw new ParseError("Empty function input parameters");

            List<Variable> range = new ArrayList<>();
            addSubMatrixVariables(inputs, range);
            if( range.size() != 1 && range.size() != 2 ) {
                throw new ParseError("Unexpected number of range variables.  1 or 2 expected");
            }
            return range;
        }

        return null;
    }
1312138
private void createSequencesConstraintsStatements(List<Statement> statements, Iterable<Sequence> sequences) {
		Statement statement = createUniqueConstraintStatement( SEQUENCE_NAME_PROPERTY, NodeLabel.SEQUENCE.name() );
		statements.add( statement );
	}
579169
def get_tokens(tokens):
    
    tokens = tokens[1:-1]
    ret = []
    start = 0
    stack = 0
    for i in range(len(tokens)):
        if tokens[i] == '(':
            if stack == 0:
                start = i
            stack += 1
        elif tokens[i] == ')':
            stack -= 1
            if stack < 0:
                raise Exception('Bracket mismatch: ' + str(tokens))
            if stack == 0:
                ret.append(get_tokens(tokens[start:i + 1]))
        else:
            if stack == 0:
                ret.append(tokens[i])
    if stack != 0:
        raise Exception('Bracket mismatch: ' + str(tokens))
    return ret
764549
public Parser<?> phrase(String... tokenNames) {
    Parser<?>[] wordParsers = new Parser<?>[tokenNames.length];
    for (int i = 0; i < tokenNames.length; i++) {
      wordParsers[i] = token(tokenNames[i]);
    }
    String phrase = Strings.join(" ", tokenNames);
    return Parsers.sequence(wordParsers).atomic().retn(phrase).label(phrase);
  }
738118
private void parseMacro( TokenList tokens , Sequence sequence ) {
        Macro macro = new Macro();

        TokenList.Token t = tokens.getFirst().next;

        if( t.word == null ) {
            throw new ParseError("Expected the macro's name after "+tokens.getFirst().word);
        }
        List<TokenList.Token> variableTokens = new ArrayList<TokenList.Token>();

        macro.name = t.word;
        t = t.next;
        t = parseMacroInput(variableTokens, t);
        for( TokenList.Token a : variableTokens ) {
            if( a.word == null) throw new ParseError("expected word in macro header");
            macro.inputs.add(a.word);
        }
        t = t.next;
        if( t == null || t.getSymbol() != Symbol.ASSIGN)
            throw new ParseError("Expected assignment");
        t = t.next;
        macro.tokens = new TokenList(t,tokens.last);

        sequence.addOperation(macro.createOperation(macros));
    }