Tokenizes according to the Penn Treebank conventions.

@param str the str
@return the string
357683
public static String tokenize(String str) {		
		str = str.replaceAll("``", " `` ");
		str = str.replaceAll("''", "  '' ");
		str = str.replaceAll("\"", "  \" ");
		str = str.replaceAll("([?!\";#$&])", " $1 ");                
		str = str.replaceAll("\\.\\.\\.", " ... ");
		str = str.replaceAll("([^.])([.])([\\])}>\"']*)\\s*$", "$1 $2$3 ");		
		str = str.replaceAll("([\\[\\](){}<>])", " $1 ");
		str = str.replaceAll("--", " -- ");

		str = str.replaceAll("$", " ");
		str = str.replaceAll("^", " ");

		//str = str.replaceAll("\"", " '' ");
		str = str.replaceAll("([^'])' ", "$1 ' ");
		str = str.replaceAll("'([sSmMdD]) ", " '$1 ");
		str = str.replaceAll("'ll ", " 'll ");
		str = str.replaceAll("'re ", " 're ");
		str = str.replaceAll("'ve ", " 've ");
		str = str.replaceAll("n't ", " n't ");
		str = str.replaceAll("'LL ", " 'LL ");
		str = str.replaceAll("'RE ", " 'RE ");
		str = str.replaceAll("'VE ", " 'VE ");
		str = str.replaceAll("N'T ", " N'T ");

		str = str.replaceAll(" ([Cc])annot ", " $1an not ");
		str = str.replaceAll(" ([Dd])'ye ", " $1' ye ");
		str = str.replaceAll(" ([Gg])imme ", " $1im me ");
		str = str.replaceAll(" ([Gg])onna ", " $1on na ");
		str = str.replaceAll(" ([Gg])otta ", " $1ot ta ");
		str = str.replaceAll(" ([Ll])emme ", " $1em me ");
		str = str.replaceAll(" ([Mm])ore'n ", " $1ore 'n ");
		str = str.replaceAll(" '([Tt])is ", " $1 is ");
		str = str.replaceAll(" '([Tt])was ", " $1 was ");
		str = str.replaceAll(" ([Ww])anna ", " $1an na ");

		//"Nicole I. Kidman" gets tokenized as "Nicole I . Kidman"
		str = str.replaceAll(" ([A-Z])\\. ", " $1 . ");
		
		
		//written by TuNC from here
	    str = str.replaceAll(",([^0-9])", ", $1");
		str = str.replaceAll("'([^'])", "' $1");   
		str = str.replaceAll("([^\\xBB])(\\xBB)", "$1 $2");					
		str = str.replaceAll("(\\u201C)([^'])", "$1 $2");    				
		str = str.replaceAll("([^'])(\\u201D)", "$1 $2");
		
    	str = str.replaceAll("\\,([^0-9])", "\\, $1"); 			                
		str = str.replaceAll("([^\\s]),([\\s])", "$1 , $2");	 			//abc,<blank> -> abc ,		
		str = str.replaceAll("([^\\s:/0-9])/([^\\s:/0-9])", "$1 / $2"); 	//exception : url http://..., date-time: 12/3/98
		str = str.replaceAll("([^\\s0-9]+)-"," $1 -"); 						//abc-xyz -> abc - xyz; exception 12-3 (date-time)
		str = str.replaceAll("-([^\\s0-9]+)","- $1");
		str = str.replaceAll("([^\\s]):([\\s])", "$1 : $2"); 				// abc:<blank> -> abc : 
	    str = str.replaceAll("([^\\s]):([^0-9]+)", "$1 : $2"); 				//abc:xyz --> abc : xyz; exception: 12:03
	    str = str.replaceAll("([^0-9]+):([^\\s])", "$1 : $2");
    	str = str.replaceAll(" -([^\\s]+)", " - $1");
    	
    	str = str.replaceAll("|", "");
    	str = str.replaceAll("[\u2026\u201C\u201D]", "");   
    	str = str.replaceAll("([^\\p{L}0-9\\.\\,:\\-/])", " $1 "); 			//tokenize all unknown characters
		str = str.replaceAll("[ \t]+", " ");
		str = str.replaceAll("^\\s+", "");	
		str = str.replaceAll("\\. \\.\\.", " ... ");
		str = str.trim();
		
		return str;
	}
1362364
public static String sqlEscapeString(final String str) {
        StringBuilder buffer = new StringBuilder(str.length() * 2);
        boolean neededEscaping = false;
        for (int i = 0; i < str.length(); i++) {
            final char c = str.charAt(i);
            if (needsEscaping((byte) c)) {
                neededEscaping = true;
                buffer.append('\\');
            }
            buffer.append(c);
        }
        return neededEscaping ? buffer.toString() : str;
    }
709789
private static final String escape(String str) {
        if (str == null) {
            return null;
        }

        StringBuilder sb = new StringBuilder(str.length() * 2);
        int len = str.length();
        for (int i = 0; i < len; i++) {
            char c = str.charAt(i);
            switch (c) {
                case TOKEN_DELIM:
                case USER_DATA_DELIM:
                case USER_ATTRIB_DELIM:
                    sb.append('\\');
                    break;
                default:
                    break;
            }
            sb.append(c);
        }
        return sb.toString();
    }
1178169
private static InputSource stringToSource(String str) {
        InputSource is = new InputSource(new StringReader(str));
        is.setEncoding("UTF-8");
        return is;
    }
210247
@Override
  public List<String> tokenize(final String text) {
    final List<String> l = new ArrayList<>();
    final StringTokenizer st = new StringTokenizer(text, nlTokenizingChars, true);
    while (st.hasMoreElements()) {
      String token = st.nextToken();
      String origToken = token;
      if (token.length() > 1) {
        if (startsWithQuote(token) && endsWithQuote(token) && token.length() > 2) {
          l.add(token.substring(0, 1));
          l.add(token.substring(1, token.length()-1));
          l.add(token.substring(token.length()-1, token.length()));
        } else if (endsWithQuote(token)) {
          int cnt = 0;
          while (endsWithQuote(token)) {
            token = token.substring(0, token.length() - 1);
            cnt++;
          }
          l.add(token);
          for (int i = origToken.length() - cnt; i < origToken.length(); i++) {
            l.add(origToken.substring(i, i + 1));
          }
        } else if (startsWithQuote(token)) {
          while (startsWithQuote(token)) {
            l.add(token.substring(0, 1));
            token = token.substring(1, token.length());
          }
          l.add(token);
        } else {
          l.add(token);
        }
      } else {
        l.add(token);
      }
    }
    return joinEMailsAndUrls(l);
  }
1049504
public static UTF8String fromString(String str) {
    return str == null ? null : fromBytes(str.getBytes(StandardCharsets.UTF_8));
  }
353028
private static Type doParse(String str) throws Exception {
        Class<?> token = PRIMITIVES.get(str);
        if (token != null) {
            return token;
        }
        int first = str.indexOf('<');
        if (first < 0) {
            return StringConvert.loadType(str);
        }
        int last = str.lastIndexOf('>');
        String baseStr = str.substring(0, first);
        Class<?> base = StringConvert.loadType(baseStr);
        String argsStr = str.substring(first + 1, last);
        List<String> splitArgs = split(argsStr);
        List<Type> types = new ArrayList<Type>();
        for (String splitArg : splitArgs) {
            Type argType;
            if (splitArg.startsWith(EXTENDS)) {
                String remainder = splitArg.substring(EXTENDS.length());
                argType = wildExtendsType(doParse(remainder));
            } else if (splitArg.startsWith(SUPER)) {
                String remainder = splitArg.substring(SUPER.length());
                argType = wildSuperType(doParse(remainder));
            } else if (splitArg.equals("?")) {
                argType = wildExtendsType(Object.class);
            } else if (splitArg.endsWith("[]")) {
                String componentStr = splitArg.substring(0, splitArg.length() - 2);
                Class<?> componentCls = StringConvert.loadType(componentStr);
                argType = Array.newInstance(componentCls, 0).getClass();
            } else if (splitArg.startsWith("[L") && splitArg.endsWith(";")) {
                String componentStr = splitArg.substring(2, splitArg.length() - 1);
                Class<?> componentCls = StringConvert.loadType(componentStr);
                argType = Array.newInstance(componentCls, 0).getClass();
            } else {
                argType = doParse(splitArg);
            }
            types.add(argType);
        }
        return newParameterizedType(base, types.toArray(new Type[types.size()]));
    }
86031
function Tokenizer(str) {

    this.str = (str || '').toString();
    this.operatorCurrent = '';
    this.operatorExpecting = '';
    this.node = null;
    this.escaped = false;

    this.list = [];

}
57968
public String trimLeading(String str) {
        if (str == null) {
            return null;
        }

        int length = str.length();
        for (int i=0; i<length; i++) {
            if (str.charAt(i) > ' ') {
                return str.substring(i);
            }
        }

        return "";
    }
240268
@Override
	public List<Token> tokenize(String s) {
		List<Token> result = new ArrayList<Token>();
		result.add(new UndefinedToken(s));

		for (PredefinedTokenDefinition predefinedTokenDefinition : _predefinedTokenDefitions) {
			Set<Pattern> patterns = predefinedTokenDefinition.getTokenRegexPatterns();
			for (Pattern pattern : patterns) {
				for (ListIterator<Token> it = result.listIterator(); it.hasNext();) {
					Token token = it.next();
					if (token instanceof UndefinedToken) {
						List<Token> replacementTokens = tokenizeInternal(token.getString(), predefinedTokenDefinition,
								pattern);
						if (replacementTokens.size() > 1) {
							it.remove();
							for (Token newToken : replacementTokens) {
								it.add(newToken);
							}
						}
					}
				}
			}
		}

		return result;
	}
122875
public function token($str) {
        $str=trim(preg_replace('/\{\{(.+?)\}\}/s',trim('\1'),
            Base::instance()->compile($str)));
        if (preg_match('/^(.+)(?<!\|)\|((?:\h*\w+(?:\h*[,;]?))+)$/s',
            $str,$parts)) {
            $str=trim($parts[1]);
            foreach (Base::instance()->split($parts[2]) as $func)
                $str=is_string($cmd=$this->filter($func))?
                    $cmd.'('.$str.')':
                    'Base::instance()->'.
                    'call($this->filter(\''.$func.'\'),['.$str.'])';
        } elseif (false !== strpos($str, '(')) {
            for ($i=0,$len=strlen($str),$newStr='',$tmp='';$i<$len;) {
                if ($str[$i] === '(') {
                    if ($tmp && array_key_exists($tmp, $this->filter)) {
                        $tmp = $this->filter[$tmp];
                    }
                    $newStr .= $tmp . '(';
                    $tmp = '';
                } elseif (in_array($str[$i], [' '])) {
                    $newStr .= $tmp . $str[$i];
                    $tmp = '';
                } else {
                    $tmp .= $str[$i];
                }
                $i++;
            }
            $newStr .= $tmp;
            $str = $newStr;
        }

        return $str;
    }
566238
public static String getPrettyFormatted(String str) {
        return String.format("%s-%s-%s-%s-%s", new Object[] { str.substring(0, 8), str.substring(8, 12), str.substring(12, 16), str.substring(16, 20), str.substring(20) });
    }
650658
public static String decode(String str) {
    if (str!=null) {
      try {
        StringBuilder sb = new StringBuilder();
        for (int idx = str.toLowerCase().indexOf("%2f"); idx>=0; idx = str.toLowerCase().indexOf("%2f")) {
          sb.append(java.net.URLDecoder.decode(str.substring(0, idx),"UTF-8")).append(str.substring(idx, idx+3));
          str = str.substring(idx+3);
        }
        sb.append(java.net.URLDecoder.decode(str,"UTF-8"));
        str = sb.toString();
      } catch (Exception ex) {
      }
    }
    return str;
  }
362780
function token($str) {
		$str=preg_replace_callback(
			'/(?<!\w)@(\w(?:[\w\.\[\]])*)/',
			function($token) {
				// Convert from JS dot notation to PHP array notation
				return '$'.preg_replace_callback(
					'/(\.\w+)|\[((?:[^\[\]]*|(?R))*)\]/',
					function($expr) {
						$fw=\Base::instance();
						return
							'['.
							($expr[1]?
								$fw->stringify(substr($expr[1],1)):
								(preg_match('/^\w+/',
									$mix=$this->token($expr[2]))?
									$fw->stringify($mix):
									$mix)).
							']';
					},
					$token[1]
				);
			},
			$str
		);
		return trim($str);
	}
613695
public static String initCap(String str) {
        assertStringNotNull(str);
        if (is_Null_or_Empty(str)) {
            return str;
        }
        if (str.length() == 1) {
            return str.toUpperCase();
        }
        final char chars[] = str.toCharArray();
        chars[0] = Character.toUpperCase(chars[0]);
        return new String(chars);
    }
1144300
static private String[] getStringList( String str )
    {
        if ( str == null )
        {
            return null;
        }
        List<String> list = new ArrayList<>();
        int i = 0;
        int length = str.length();
        StringBuffer sb = null;
        while ( i < length )
        {
            char ch = str.charAt( i );
            if ( ch == ' ' )
            {
                // A space was hit. Add string to list
                if ( sb != null )
                {
                    list.add( sb.toString() );
                    sb = null;
                }
            }
            else if ( ch == '\\' )
            {
                // It is a delimiter. Add next character
                if ( i + 1 < length )
                {
                    ch = str.charAt( ++i );
                    if ( sb == null )
                    {
                        sb = new StringBuffer();
                    }
                    sb.append( ch );
                }
            }
            else
            {
                if ( sb == null )
                {
                    sb = new StringBuffer();
                }
                sb.append( ch );
            }
            i++; // Next character
        }
        // Make sure to add the last part to the list too
        if ( sb != null )
        {
            list.add( sb.toString() );
        }
        if ( list.size() == 0 )
        {
            return null;
        }
        String[] results = new String[list.size()];
        return list.toArray( results );
    }
424904
public static List<Word> parseWithoutStopWords(String str) {
        List<Word> words = WordSegmenter.seg(str, SegmentationAlgorithm.MaxNgramScore);
        //词性标注
        PartOfSpeechTagging.process(words);
        return words;
    }