Tokenize string based on rules

@param tokenString
@param delimiters
@return
163482
public static List<String> wonderfulTokenizer(String tokenString, String[] delimiters) {
		List<String> tokens = new ArrayList<String>();
		String currentToken = "";
		for(int i = 0; i < tokenString.length(); i++) {
			String _currItem = String.valueOf(tokenString.charAt(i));
			if(_currItem.trim().isEmpty()) continue;

			String delimiterFound = "";
			for(String d: delimiters) {
				if(d.startsWith(_currItem)) {
					if(d.length() > 1) {
						if(d.equals(tokenString.substring(i, i + d.length()))) {
							delimiterFound = d;
							i = i + (d.length() - 1);
						}
					} else delimiterFound = d;
				}
				if(!delimiterFound.isEmpty()) break;
			}

			if(!delimiterFound.isEmpty()) {
				if(!currentToken.isEmpty()) {
					tokens.add(currentToken);
					currentToken = "";
				}
				tokens.add(delimiterFound);
			} else currentToken += _currItem;
		}
		if(!currentToken.isEmpty()) tokens.add(currentToken);

		return tokens;
	}
736635
public static AuthenticationCookie fromDelimitedString(final String delimitedString) {
    Preconditions.checkArgument(delimitedString != null && delimitedString.length() > 0,
                                "delimitedString cannot be empty");

    final String[] cookieElements = delimitedString.split(DELIMITER);
    Preconditions.checkArgument(cookieElements.length == 4, "delimitedString should contain exactly 4 elements");

    return new AuthenticationCookie(
      cookieElements[0],
      DateTime.parse(cookieElements[1], DATE_TIME_FORMATTER),
      cookieElements[2],
      cookieElements[3]);
  }
473536
public static String getFirstToken(String string, String token) {
        if ((string != null) && string.contains(token)) {
            return string.split(token)[0];
        }
        return string;
    }
1137115
public static String join(String _delimiter, List<String> _strings) {
        if (_strings == null) {
            return null;
        }
        if (_delimiter == null) {
            _delimiter = "";
        }
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < _strings.size(); i++) {
            sb.append(_strings.get(i));
            if (i < _strings.size() - 1) { // only append delimiter if this is not the last token
                sb.append(_delimiter);
            }
        }
        return sb.toString();
    }
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
1024967
public List<Tokenizer.Token> tokenize(String sentence) {
      Sequence seq = Sequence.create(Chars.asList(sentence.toCharArray())
                                          .stream()
                                          .map(Object::toString));
      Labeling labels = model.label(seq);

      int n = sentence.length();
      int[][] spans = new int[n + 1][2];
      double[] best = new double[n + 1];
      best[0] = 1d;

      for (int i = 1; i <= n; i++) {
         for (int j = Math.max(0, i - maxSpanSize); j < i; j++) {
            int w = i - j;
            String span = sentence.substring(j, j + w);
            double score = scoreWord(j, j + w, span, labels);
            double segmentScore = multi(best[i - w], score);
            logFiner("'{0}' > '{1}' : {2} : {3}",
                     sentence.substring(spans[i - w][0], spans[i - w][1]),
                     span,
                     segmentScore,
                     best[i]);
            if (segmentScore >= best[i]) {
               best[i] = segmentScore;
               spans[i][0] = j;
               spans[i][1] = j + w;
            }
         }
      }
      int i = n;
      LinkedList<Tokenizer.Token> tokens = new LinkedList<>();
      while (i > 0) {
         String token = StringUtils.trim(sentence.substring(spans[i][0], spans[i][1]));
         if (!StringUtils.isNullOrBlank(token)) {
            tokens.addFirst(
               new Tokenizer.Token(token, TokenType.UNKNOWN, spans[i][0], spans[i][0] + token.length(), 0));
         }
         i = i - (spans[i][1] - spans[i][0]);
      }
      i = 0;
      for (Tokenizer.Token token : tokens) {
         token.index = i;
         i++;
      }

      return tokens;
   }
1082459
public static List<String> split(String string, String delimiter) {
        return split(string, delimiter, true);
    }
608148
function(tokens) {
            	if(typeof tokens === "string") {
            		tokens = [tokens];
            	}
                    var results = [];
            	var rule_count = rules$3.length;
            	var num_tokens = tokens.length;
                    var i, token, r, rule;
                
                    for (i = 0; i < num_tokens; i++) {
                        token = tokens[i];
                        // Check the conversion table
                        if (conversionTable[token.toLowerCase()]) {
                            results = results.concat(conversionTable[token.toLowerCase()].split(/\W+/));
                        }
                        
                        // Apply the rules
                        else {
                            var matched = false;
                            for ( r = 0; r < rule_count; r++) {
                                rule = rules$3[r];
                                if (token.match(rule.regex)) {
                                    results = results.concat(token.replace(rule.regex, rule.output).split(/\W+/));
                                    matched = true;
                                    break;
                                }
                            }
                            if (!matched) {
                                results.push(token);
                            }
                        }
                    }

            	return results;
            }
733319
static function Tokenize($string, $splitChars = ' %$§&/()?!.:;\#\'"')
    {
       $tokens = array();

       $tok = strtok($string,  $splitChars);
       while ($tok !== false)
       {
            array_push($tokens, $tok);
            $tok = strtok($splitChars);
       }
       return $tokens;
    }
389486
public static void forEachToken(String string, String separator, Procedure<String> procedure)
    {
        for (StringTokenizer stringTokenizer = new StringTokenizer(string, separator); stringTokenizer.hasMoreTokens(); )
        {
            String token = stringTokenizer.nextToken();
            procedure.value(token);
        }
    }
653831
public static String join(String delimiter, Iterable<String> strings) {
    StringBuilder sb = new StringBuilder();
    for (String item : strings) {
      if (sb.length() > 0) sb.append(delimiter);
      sb.append(item);
    }
    return sb.toString();
  }
1033430
private String [] split (final String string, final String delim)
  {
    final List <String> result = new ArrayList <> ();

    final StringTokenizer stringTokenizer = new StringTokenizer (string, delim);
    while (stringTokenizer.hasMoreTokens ())
    {
      result.add (stringTokenizer.nextToken ());
    }

    return result.toArray (new String [result.size ()]);
  }
874473
private static String[] split(String value, String delimeter) {
        StringTokenizer st = new StringTokenizer(value, delimeter);
        String[] res = new String[st.countTokens()];
        for (int i = 0; st.hasMoreTokens(); i++) {
            res[i] = st.nextToken();
        }
        return res;
    }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
683791
public String toPOSString(char delimiter) {
      return tokens()
                .stream()
                .map(t -> t.toString() + delimiter + t
                                                        .get(Types.PART_OF_SPEECH)
                                                        .as(POS.class, POS.ANY)
                                                        .asString())
                .collect(Collectors.joining(" "));
   }
240268
@Override
	public List<Token> tokenize(String s) {
		List<Token> result = new ArrayList<Token>();
		result.add(new UndefinedToken(s));

		for (PredefinedTokenDefinition predefinedTokenDefinition : _predefinedTokenDefitions) {
			Set<Pattern> patterns = predefinedTokenDefinition.getTokenRegexPatterns();
			for (Pattern pattern : patterns) {
				for (ListIterator<Token> it = result.listIterator(); it.hasNext();) {
					Token token = it.next();
					if (token instanceof UndefinedToken) {
						List<Token> replacementTokens = tokenizeInternal(token.getString(), predefinedTokenDefinition,
								pattern);
						if (replacementTokens.size() > 1) {
							it.remove();
							for (Token newToken : replacementTokens) {
								it.add(newToken);
							}
						}
					}
				}
			}
		}

		return result;
	}
1297535
public static String asTokenSeparatedValues(String token, Collection<String> strings) {
        StringBuilder newString = new StringBuilder();
        boolean first = true;
        for(String str : strings) {
            if(! first) {
                newString.append(token);
            }
            first = false;
            newString.append(str);
        }
        return newString.toString();
    }