Utility method which parses tokens until a semicolon is found or there are no more tokens.

@param tokens the {@link DdlTokenStream} representing the tokenized DDL content; may not be null
@return the parsed string
@throws ParsingException
342181
protected String parseUntilSemiColon( DdlTokenStream tokens ) throws ParsingException {
        StringBuilder sb = new StringBuilder();

        boolean lastTokenWasPeriod = false;
        while (tokens.hasNext() && !tokens.matches(SEMICOLON)) {
            String thisToken = tokens.consume();
            boolean thisTokenIsPeriod = thisToken.equals(PERIOD);
            boolean thisTokenIsComma = thisToken.equals(COMMA);
            if (lastTokenWasPeriod || thisTokenIsPeriod || thisTokenIsComma) {
                sb.append(thisToken);
            } else {
                sb.append(SPACE).append(thisToken);
            }
            if (thisTokenIsPeriod) {
                lastTokenWasPeriod = true;
            } else {
                lastTokenWasPeriod = false;
            }
        }

        return sb.toString();
    }
655761
@Override
    public AstNode handleUnknownToken( DdlTokenStream tokens,
                                       String tokenValue ) throws ParsingException {
        if (tokenValue.equals("/")) {
            return nodeFactory().node("backslashTerminator", getRootNode(), TYPE_BACKSLASH_TERMINATOR);
        }

        return null;
    }
1299759
@Override
    protected Source parseFrom( TokenStream tokens,
                                TypeSystem typeSystem ) {
        Position firstSourcePosition = tokens.nextPosition();
        Source source = super.parseFrom(tokens, typeSystem);
        if (tokens.matches(',') && source instanceof NamedSelector) {
            NamedSelector selector = (NamedSelector)source;
            JoinableSources joinedSources = new JoinableSources(selector, firstSourcePosition);
            while (tokens.canConsume(',')) {
                // This is a JCR-SQL-style JOIN ...
                Position nextSourcePosition = tokens.nextPosition();
                NamedSelector nextSource = parseNamedSelector(tokens, typeSystem);
                joinedSources.add(nextSource, nextSourcePosition);
            }
            source = joinedSources;
        }
        return source;
    }
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
238661
private void convertTokenDefinitions(Map<String, ParseTreeNode> helpers,
	    Map<String, ParseTreeNode> tokens) throws GrammarException,
	    TreeException {
	tokenDefinitions = new TokenDefinitionSet();
	for (ParseTreeNode tokenDefinitionAST : parserTree.getChild("Tokens")
		.getChild("TokenDefinitions").getChildren("TokenDefinition")) {
	    TokenDefinition convertedTokenDefinition = getTokenDefinition(
		    tokenDefinitionAST, helpers, tokens);
	    tokenDefinitions.addDefinition(convertedTokenDefinition);
	}
    }
1299636
public final String[][] getAllPosTags(final String[] tokens) {
    final String[][] allPosTags = this.posTagger.tag(13, tokens);
    return allPosTags;
  }
1158046
public static String[] getTypesFromSpans(final Span[] spans,
      final String[] tokens) {
    final List<String> tagsList = new ArrayList<String>();
    for (final Span span : spans) {
      tagsList.add(span.getType());
    }
    return tagsList.toArray(new String[tagsList.size()]);
  }
323913
private String getSentenceFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String token : tokens) {
      sb.append(token).append(" ");
    }
    final String sentence = sb.toString();
    return sentence;
  }
877721
public static List<MimeType> parseMimeTypes(String mimeTypes) {
        if (!Strings.isNullOrEmpty(mimeTypes)) {
            return Collections.emptyList();
        }
        String[] tokens = mimeTypes.split(",\\s*");
        List<MimeType> result = new ArrayList<MimeType>(tokens.length);
        for (String token : tokens) {
            result.add(parseMimeType(token));
        }
        return result;
    }
1352541
public static String getStringFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String tok : tokens) {
      sb.append(tok).append(" ");
    }
    return sb.toString().trim();
  }
629177
public static List<TokensLine> convert(List<TokenEntry> tokens) {
    List<TokensLine> result = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    int startLine = Integer.MIN_VALUE;
    int startIndex = 0;
    int currentIndex = 0;
    for (TokenEntry token : tokens) {
      if (token != TokenEntry.EOF) {
        String value = token.getValue();
        int line = token.getBeginLine();
        if (line != startLine) {
          addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
          startIndex = currentIndex + 1;
          startLine = line;
        }
        currentIndex++;
        sb.append(value);
      }
    }
    addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
    return result;
  }
1023475
public static void normalizeTokens(final List<List<Token>> tokens,
      final String lang) {
    for (final List<Token> sentence : tokens) {
      Normalizer.convertNonCanonicalStrings(sentence, lang);
      Normalizer.normalizeQuotes(sentence, lang);
      Normalizer.normalizeDoubleQuotes(sentence, lang);
    }
  }
812212
@SuppressWarnings({ "PMD.AvoidInstantiatingObjectsInLoops",
            "PMD.LongVariable", "PMD.ExcessiveMethodLength",
            "PMD.DataflowAnomalyAnalysis", "PMD.NcssMethodCount",
            "PMD.NPathComplexity" })
    private List<Token> parseTokenLogFileDefinition(final String tokenLogFormat) {

        // Add all available parsers
        final List<TokenParser> tokenParsers = createAllTokenParsers();
        final List<Token> tokens = new ArrayList<>(50);

        // We first change all the references to headers to lowercase
        // because we must handle these as "case insensitive"
        String cleanedTokenLogFormat = cleanupLogFormat(tokenLogFormat);

        // Now we let all tokens figure out if they are present in here
        for (TokenParser tokenParser : tokenParsers) {
            List<Token> newTokens = tokenParser.getTokens(cleanedTokenLogFormat);
            if (newTokens != null) {
                tokens.addAll(newTokens);
            }
        }

        // We now have a full list of all matched tokens
        // ---------------------------------------
        // We sort them by position of the token in the format specifier
        tokens.sort(new TokenSorterByStartPos());

        // First we take out the duplicates with a lower prio(=relevance score)
        final List<Token> kickTokens = new ArrayList<>(50);
        Token prevToken = null;
        for (Token token : tokens) {
            if (prevToken==null){
                prevToken=token;
                continue;
            }

            if (prevToken.getStartPos() == token.getStartPos()) {
                if (prevToken.getLength() == token.getLength()) {
                    if (prevToken.getPrio() < token.getPrio()) {
                        kickTokens.add(prevToken);
                    } else {
                        kickTokens.add(token);
                    }
                } else {
                    if (prevToken.getLength() < token.getLength()) {
                        kickTokens.add(prevToken);
                    } else {
                        kickTokens.add(token);
                    }
                }
            } else {
                // Sometimes we find that a part of a token matches another token aswell.
                // Example: %{%H}t    Custom Timeformat (only the hour) also matches the protocol token.
                // So we kick them of they overlap
                if (prevToken.getStartPos() + prevToken.getLength() > token.getStartPos()) {
                    kickTokens.add(token);
                    continue;
                }
            }
            prevToken=token;

        }

        tokens.removeAll(kickTokens);

        final List<Token> allTokens = new ArrayList<>(50);
        // We now look for the holes and add "FIXED STRING" tokens
        int tokenBegin;
        int tokenEnd = 0;
        for (Token token : tokens) {
            tokenBegin = token.getStartPos();
            // Space between the begin of the next token and the end of the previous token?
            if (tokenBegin - tokenEnd > 0) {
                String separator = cleanedTokenLogFormat.substring(tokenEnd, tokenBegin);
                Token fixedStringToken = new FixedStringToken(separator, tokenBegin, tokenBegin - tokenEnd, 0);
                allTokens.add(fixedStringToken);
            }
            allTokens.add(token);
            tokenEnd = tokenBegin + token.getLength();
        }

        int logFormatLength = cleanedTokenLogFormat.length();
        if (tokenEnd < logFormatLength) {
            String separator = cleanedTokenLogFormat.substring(tokenEnd);
            Token fixedStringToken = new FixedStringToken(separator, tokenEnd, cleanedTokenLogFormat.length() - tokenEnd, 0);
            allTokens.add(fixedStringToken);
        }

        return allTokens;
    }
544560
private void convertTokenDefinitions() throws GrammarException,
	    TreeException {
	tokenVisibility.clear();
	Map<String, ParseTreeNode> helpers = getHelperTokens();
	Map<String, ParseTreeNode> tokens = getTokens();
	convertTokenDefinitions(helpers, tokens);
    }
937825
protected void parseColumns( DdlTokenStream tokens,
                                 AstNode tableNode,
                                 boolean isAlterTable ) throws ParsingException {
        String tableElementString = getTableElementsString(tokens, false);

        DdlTokenStream localTokens = new DdlTokenStream(tableElementString, DdlTokenStream.ddlTokenizer(false), false);

        localTokens.start();

        StringBuilder unusedTokensSB = new StringBuilder();

        do {
            if (isColumnDefinitionStart(localTokens)) {
                parseColumnDefinition(localTokens, tableNode, isAlterTable);
            } else {
                // THIS IS AN ERROR. NOTHING FOUND.
                // NEED TO absorb tokens
                unusedTokensSB.append(SPACE).append(localTokens.consume());
            }
        } while (localTokens.canConsume(COMMA));

        if (unusedTokensSB.length() > 0) {
            String msg = DdlSequencerI18n.unusedTokensParsingColumnDefinition.text(tableNode.getName());
            DdlParserProblem problem = new DdlParserProblem(Problems.WARNING, getCurrentMarkedPosition(), msg);
            problem.setUnusedSource(unusedTokensSB.toString());
            addProblem(problem, tableNode);
        }
    }
1374312
@Override
    protected void initializeTokenStream( DdlTokenStream tokens ) {
        super.initializeTokenStream(tokens);
        tokens.registerKeyWords(CUSTOM_KEYWORDS);
        tokens.registerStatementStartPhrase(ALTER_PHRASES);
        tokens.registerStatementStartPhrase(CREATE_PHRASES);
        tokens.registerStatementStartPhrase(DROP_PHRASES);
        tokens.registerStatementStartPhrase(SET_PHRASES);
        tokens.registerStatementStartPhrase(MISC_PHRASES);
    }
1298584
protected long parseLong( DdlTokenStream tokens,
                              DataType dataType ) {
        String value = consume(tokens, dataType, false);
        return parseLong(value);
    }