Creates an almost unique String code for a list of tokens. This code is
used to improve search time when looking for potential matching patterns.

@param tokens
@return
766981
private String getPatternCode(final List<Token> tokens) {
        final StringBuilder sb = new StringBuilder();
        sb.append(tokens.size());
        for (final Token token : tokens) {
            sb.append(token.getType().ordinal());
        }
        return sb.toString();
    }
1215488
public List<String> getTokenList() {
        checkTokenized();
        final List<String> list = new ArrayList<>(tokens.length);
        list.addAll(Arrays.asList(tokens));
        return list;
    }
495843
public static ImmutableList<UnicodeFriendlyString> unicodeFriendlyList(Iterable<String> strings) {
    final ImmutableList.Builder<UnicodeFriendlyString> ret = ImmutableList.builder();

    for (final String s : strings) {
      ret.add(unicodeFriendly(s));
    }

    return ret.build();
  }
1242510
private static String[] tokenize(String string) {
		if (string.indexOf("?") == -1 && string.indexOf("*") == -1) {
			return new String[] { string };
		}

		char[] textArray = string.toCharArray();
		List<String> tokens = new ArrayList<String>();
		StringBuilder tokenBuilder = new StringBuilder();

		for (int i = 0; i < textArray.length; i++) {
			if (textArray[i] != '?' && textArray[i] != '*') {
				// collect non wild card characters
				tokenBuilder.append(textArray[i]);
				continue;
			}

			if (tokenBuilder.length() != 0) {
				tokens.add(tokenBuilder.toString());
				tokenBuilder.setLength(0);
			}
			if (textArray[i] == '?') {
				tokens.add("?");
			} else if (tokens.size() == 0 || (i > 0 && tokens.get(tokens.size() - 1).equals("*") == false)) {
				tokens.add("*");
			}
		}

		if (tokenBuilder.length() != 0) {
			tokens.add(tokenBuilder.toString());
		}

		return (String[]) tokens.toArray(new String[tokens.size()]);
	}
1006837
public static StringBuilder UNDER_SCORE(final StringBuilder sb, final String... tokens) {
        return Strings.join(sb, "_", allToUpper(tokens));
    }
409989
protected <T extends TokenBase> List<T> createTokenList(String text) {

        if (!split) {
            return createTokenList(0, text);
        }

        List<Integer> splitPositions = getSplitPositions(text);

        if (splitPositions.isEmpty()) {
            return createTokenList(0, text);
        }

        ArrayList<T> result = new ArrayList<>();

        int offset = 0;

        for (int position : splitPositions) {
            result.addAll(this.<T>createTokenList(offset, text.substring(offset, position + 1)));
            offset = position + 1;
        }

        if (offset < text.length()) {
            result.addAll(this.<T>createTokenList(offset, text.substring(offset)));
        }

        return result;
    }
1053880
protected static char[] flags(final CharSequence... flagStrings) {
        final StringBuilder sb = new StringBuilder();
        for (CharSequence flags: flagStrings) sb.append(flags);
        return flags(sb.toString());
    }
325657
public static String[] unique(String[] arrays) {
        List<String> list = new ArrayList<>(arrays.length);
        for (String arr : arrays) {
            if (!list.contains(arr)) {
                list.add(arr);
            }
        }
        String[] res = new String[list.size()];
        int i = 0;
        for (String arr : list) {
            res[i++] = arr;
        }
        return res;
    }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
254863
static List<String> getCommaTokens(String inString) {
        String input = inString.trim();
        List<String> tokens = new ArrayList<String>();
        String token;
        while (0 < input.length()) {
            if (COMMA == input.charAt(0)) {
                input = input.substring(1);
            }
            token = getNextCommaToken(input);

            input = input.substring(token.length());
            tokens.add(token.trim());
        }

        return tokens;

    }
792225
public static Token[] optimize(Token[] tokens) {
        if (tokens == null) {
            return null;
        }

        String firstVal = null;
        String lastVal = null;

        if (tokens.length == 1) {
            if (tokens[0].getType() == TokenType.TEXT) {
                firstVal = tokens[0].getDefaultValue();
            }
        } else if (tokens.length > 1) {
            if (tokens[0].getType() == TokenType.TEXT) {
                firstVal = tokens[0].getDefaultValue();
            }
            if (tokens[tokens.length - 1].getType() == TokenType.TEXT) {
                lastVal = tokens[tokens.length - 1].getDefaultValue();
            }
        }

        if (firstVal != null) {
            String text = trimLeadingWhitespace(firstVal);
            if (!Objects.equals(firstVal, text)) {
                tokens[0] = new Token(text);
            }
        }

        if (lastVal != null && !lastVal.isEmpty()) {
            String text = trimTrailingWhitespace(lastVal);
            if (!Objects.equals(lastVal, text)) {
                tokens[tokens.length - 1] = new Token(text);
            }
        }

        return tokens;
    }
54699
public final List<Morpheme> getMorphemesFromStrings(
      final List<String> posTags, final String[] tokens) {
    final List<Morpheme> morphemes = new ArrayList<Morpheme>();
    for (int i = 0; i < posTags.size(); i++) {
      final String word = tokens[i];
      final String tag = posTags.get(i);
      final Morpheme morpheme = this.morphoFactory.createMorpheme(word, tag);
      morphemes.add(morpheme);
    }
    return morphemes;
  }
877721
public static List<MimeType> parseMimeTypes(String mimeTypes) {
        if (!Strings.isNullOrEmpty(mimeTypes)) {
            return Collections.emptyList();
        }
        String[] tokens = mimeTypes.split(",\\s*");
        List<MimeType> result = new ArrayList<MimeType>(tokens.length);
        for (String token : tokens) {
            result.add(parseMimeType(token));
        }
        return result;
    }
530895
public static boolean stringInPatterns(String s, List<Pattern> patterns) {
    for (Pattern pattern : patterns) {
      if (pattern.matcher(s).matches()) {
        return true;
      }
    }
    return false;
  }
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
313464
public List<Match<E>> findAll(List<E> tokens) {
        List<Match<E>> results = new ArrayList<Match<E>>();

        int start = 0;
        Match<E> match;
        do {
            match = this.find(tokens, start);

            if (match != null) {
                start = match.endIndex();

                // match may be empty query string has all optional parts
                if (!match.isEmpty()) {
                    results.add(match);
                }
            }
        } while (match != null);

        return results;
    }
1158046
public static String[] getTypesFromSpans(final Span[] spans,
      final String[] tokens) {
    final List<String> tagsList = new ArrayList<String>();
    for (final Span span : spans) {
      tagsList.add(span.getType());
    }
    return tagsList.toArray(new String[tagsList.size()]);
  }