Tokenizes the given text into normalized uppercased tokens

@param String text Text you want to parse and tokenize
@param Function callback Callback gets called with this parameters function(error, String[])
@api public
778837
function tokenizeText(text, callback) {
    "use strict";
    processTokens(tokenizer.tokenize(text) || [], function (error, tokens) {
        process.nextTick(function () {
            callback(error, tokens);
        });
    });
}
318931
@Override
    public void setAsText(String text) throws IllegalArgumentException {
        if(text == null){
            setValue(null);
            return;
        }
        setValue(tokenize(text));
    }
1043422
function(text) {
      if(text == null) return this.node.textContent + (this.dom.newLined ? '\n' : '')

      typeof text === 'function' ? text.call(this, this) : this.plain(text)

      return this
    }
470376
public ChannelFuture respondJsonPText(Object text, String function) throws Exception {
        return respondJs(function + "(" + text + ");\r\n");
    }
601657
public static function replaceStringsAndComments($text)
    {
        if (is_string($text)) {
            // Fast pre-check, to optimize performance
            if (strpos($text, "'") !== false || strpos($text, '"') !== false) {
                // Replace all strings in quotes (also in comments, but that's okay - removed in next step)
                $replaceStringCallback = function ($matches) {
                    return self::addStringReplacement($matches[1] . $matches[2] . $matches[1]);
                };
                $text = preg_replace('/\r/', '', $text);
                $text = preg_replace('/\\\\\n/', '_STRING_CSSLINEBREAK_', $text);
                $text = preg_replace('/\\\\"/', '_STRING_ESCAPEDDOUBLEQUOTE_', $text);
                $text = preg_replace('/\\\\\'/', '_STRING_ESCAPEDSINGLEQUOTE_', $text);
                $text = preg_replace_callback('/("|\')(.*?)\g{1}/', $replaceStringCallback, $text);
            }

            if (strpos($text, "data:") !== false) {
                // Replace all data URIs
                // (that are not in quotes and therefor were not replaced by the previous check)
                $replaceDataUriCallback = function ($matches) {
                    return self::addStringReplacement($matches[1]);
                };
                $text = preg_replace_callback(
                    '/(data:(?:[^;,]+)?(?:;charset=[^;,]+)?' .
                    '(?:;base64,(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?|,[-.a-zA-Z0-9_%]+))/',
                    $replaceDataUriCallback,
                    $text
                );
            }

            // Fast pre-check, to optimize performance
            if (strpos($text, "*") !== false) {
                // Strip multi-line comments (after string replace to keep comments in strings)
                $replaceCommentCallback = function ($matches) {
                    return self::addCommentReplacement($matches[1]);
                };
                $text = preg_replace_callback('/(\/\*.*?\*\/)/', $replaceCommentCallback, $text);
            }

            return $text;
        } else {
            throw new \InvalidArgumentException(
                "Invalid type '" . gettype($text). "' for argument 'text' given."
            );
        }
    }
638256
public static Token[] parse(String text, boolean optimize) {
        if (text == null) {
            return null;
        }
        if (text.length() == 0) {
            Token t = new Token(text);
            return new Token[] { t };
        }
        Token[] tokens = null;
        List<Token> tokenList = Tokenizer.tokenize(text, optimize);
        if (!tokenList.isEmpty()) {
            tokens = tokenList.toArray(new Token[0]);
            if (optimize) {
                tokens = Tokenizer.optimize(tokens);
            }
        }
        return tokens;
    }
352587
public String senTokenize(String text){
		String ret = text;
		
		if (isTokenization){
			ret = PennTokenizer.tokenize(text);
		}
		
		return ret.trim();
	}
991777
function ( text ){
		   var out = [text],
			   chars = ['-','+','z','Z'],
			   i = chars.length;

			while (i--) {
			  if(text.indexOf(chars[i]) > -1){
				  out[0] = text.slice( 0, text.indexOf(chars[i]) );
				  out.push( text.slice( text.indexOf(chars[i]) ) );
				  break;
			   }
			}
		   return out;
		}
1085129
def normalize_text(text: str) -> str:
    
    return ' '.join([token
                     for token in text.lower().strip(STRIPPED_CHARACTERS).split()
                     if token not in IGNORED_TOKENS])
131087
function processTokens(tokens, callback) {
    "use strict";
    var processFunctions = underscore.map(tokens, function (word) {
        return function (cb) {
            tokenizeWord(word, cb);
        };
    });
    async.parallel(
        processFunctions,
        function (error, results) {
            if (error) {
                callback(error, null);
            } else {
                var normalizedTokens = [];
                results.forEach(function (tokens) {
                    normalizedTokens = normalizedTokens.concat(tokens);
                });
                normalizedTokens.sort();
                normalizedTokens = underscore.uniq(normalizedTokens, true);
                callback(null, normalizedTokens);
            }
        }
    );
}
449875
public function parse(string $text) : string
    {
        $tokens = $this->tokenizer->tokenize($text);
        $segments = $this->createSegments($tokens);
        $nodes = $this->createTree($segments);
        return $this->generator->generateMarkup($nodes);
    }
753930
public void setValue(String text) {
        Token[] tokens = TokenParser.makeTokens(text, isTokenize());
        setValue(tokens);
    }
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
1349036
function parse(text) {
    const textLineByLine = text.split(LINEBREAKE_MARK);
    // it should be alternately Str and Break
    let startIndex = 0;
    const lastLineIndex = textLineByLine.length - 1;
    const isLasEmptytLine = (line, index) => {
        return index === lastLineIndex && line === "";
    };
    const isEmptyLine = (line, index) => {
        return index !== lastLineIndex && line === "";
    };
    const children = textLineByLine.reduce(function(result, currentLine, index) {
        const lineNumber = index + 1;
        if (isLasEmptytLine(currentLine, index)) {
            return result;
        }
        // \n
        if (isEmptyLine(currentLine, index)) {
            const emptyBreakNode = createBRNode(lineNumber, startIndex);
            startIndex += emptyBreakNode.raw.length;
            result.push(emptyBreakNode);
            return result;
        }

        // (Paragraph > Str) -> Br?
        const strNode = parseLine(currentLine, lineNumber, startIndex);
        const paragraph = createParagraph([strNode]);
        startIndex += paragraph.raw.length;
        result.push(paragraph);
        if (index !== lastLineIndex) {
            const breakNode = createEndedBRNode(paragraph);
            startIndex += breakNode.raw.length;
            result.push(breakNode);
        }
        return result;
    }, []);
    return {
        type: Syntax.Document,
        raw: text,
        range: [0, text.length],
        loc: {
            start: {
                line: 1,
                column: 0
            },
            end: {
                line: textLineByLine.length,
                column: textLineByLine[textLineByLine.length - 1].length
            }
        },
        children
    };
}
1127113
function parse(text) {
  const compCount = U.countComponents(text);
  if (!compCount) {
    return; // undefined
  }

  const len = text.length,
    components = new Array(compCount);

  let multiplier = 1,
    pos = 0,
    nextComp = 0;

  //console.log(`parsing unit ${text}:`);
  while (pos < len) {
    let ch = text.charCodeAt(pos);

    if (ch === U.slash) {
      multiplier *= -1;
      pos++;
    } else if (ch === U.asterisk) {
      pos++;
    }

    let s = pos;
    for (; pos < len; pos++) {
      ch = text.charCodeAt(pos);

      if (!U.isLetter(ch)) {
        break;
      }
    }

    const name = text.substring(s, pos);
    const unitType = bySid[name];
    if (!unitType) {
      throw new Error(`Unknown unit type "${name}" in "${text}"`);
    }

    if (pos < len && text.charCodeAt(pos) === U.caret) {
      pos++;
    }

    s = pos;
    if (pos < len && text.charCodeAt(pos) === U.minus) {
      pos++;
    }

    while (pos < len && U.isDigit(text.charCodeAt(pos))) {
      pos++;
    }

    const degreeText = text.substring(s, pos);

    let degree;
    if (!degreeText.length) {
      degree = 1;
    } else {
      degree = Number.parseInt(degreeText);
      if (isNaN(degree)) {
        throw new Error(`Illegal degree "${degreeText}" in "${text}"`);
      }
    }

    degree *= multiplier;

    let ci = 0;
    for (; ci < nextComp; ci++) {
      const comp = components[ci];

      if (comp.unitType === unitType) {
        comp.degree += degree;
        break;
      }
    }

    if (ci >= nextComp) {
      components[nextComp++] = new UnitTypeDegree(unitType, degree);
    }

    //console.log(` + ${name} ==> ${degree}`);
  }

  components.length = nextComp;
  return parseComponents(components);
}
91927
function tokenize (str) {
  let pos = 0
  let row = 1
  let col = 1
  let newline = true
  let state = CONTENT
  let start = 0
  let consumingSameLineContent = false
  let escapedInlineCounter = 0
  let escapedParamCounter = 0
  let consumingEscapedParams = false
  let consumingUnparsed = false // true when in an @@ block
  let lastInlineContentEndPos = void (0)
  let unparsedIndentStart = void (0)

  const indent = [0]
  const tokens = []

  function emit (type, value) {
    if (arguments.length > 1) {
      tokens.push({type: type, value: value})
    } else {
      tokens.push({type: type})
    }
  }

  function consume (type, next) {
    emit(type, str.substring(start, pos))
    state = next
    start = pos + 1
  }

  function consumeIfNonEmpty (type, next, escaped) {
    let v = str.substring(start, pos)
    if (v.length > 0) {
      if (escaped) {
        const l = escaped.length
        for (let i = 0; i < l; i++) {
          const es = escaped[i]
          v = v.replace(es, es[1])
        }
      }
      emit(type, v)
    }
    state = next
    start = pos + 1
  }

  function shuffleIfNext (character) {
    if (str[pos + 1] === character) {
      pos++
      start++
      return true
    } else {
      return false
    }
  }

  function err (msg) {
    let start = str.lastIndexOf('\n', pos - 1)
    start = str.lastIndexOf('\n', start - 1)
    start = str.lastIndexOf('\n', start - 1)

    const nextNewline = str.indexOf('\n', pos)
    const errorLineEnd = nextNewline === -1 ? pos : nextNewline

    let end = str.indexOf('\n', errorLineEnd + 1)
    if (end === -1) {
      end = str.length - 1
    } else {
      end = str.indexOf('\n', end + 1)
      if (end === -1) {
        end = str.length - 1
      } else {
        end = str.indexOf('\n', end + 1)
        if (end === -1) {
          end = str.length - 1
        }
      }
    }

    const indentSpaces = (new Array(col)).join(' ')

    const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg
    const context = str.substring(start, errorLineEnd) + '\n' + indentSpaces + '^^^^^^^^\n' + str.substring(errorLineEnd + 1, end)

    throw new ParseError(message + '\n' + context, context, str, row, col, msg, pos)
  }

  while (pos < str.length) {
    // indentation and comment handling.
    if (newline) {
      while (newline && pos < str.length) {
        let ind = 0
        while (str[pos + ind] === ' ' && pos + ind < str.length) {
          ind++
        }

        // if it was an empty content line, then add it to the list of empty
        // content to emit on the next non empty line
        if (str[pos + ind] === '\n') {
          pos += ind + 1
          row++
          emit('EMPTY_CONTENT', str.substring(start, pos - 1))
        } else if (str[pos + ind] !== '#' || consumingUnparsed) {
          if (ind > indent[indent.length - 1]) {
            emit('INDENT', ind - indent[indent.length - 1])
            indent.push(ind)
          } else if (ind < indent[indent.length - 1]) {
            while (indent[indent.length - 1] > ind) {
              const prev = indent.pop()
              emit('DEDENT', prev - indent[indent.length - 1])

              if (consumingUnparsed && unparsedIndentStart >= indent[indent.length - 1]) {
                consumingUnparsed = false
              }
            }

            if (indent.length > 0 && (indent[indent.length - 1] !== ind)) {
              err('indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent[indent.length - 1] + ' spaces was expected')
            }
          }
          pos += ind

          if (str[pos] === '\\' && str[pos + 1] === '#') {
            pos++
          }
          newline = false
        } else {
          while (str[pos] !== '\n' && pos < str.length) {
            pos++
          }
          emit('COMMENT', str.substring(start + ind + 1, pos))
          pos++
          row++
        }

        start = pos
      }
    }

    if (pos >= str.length) {
      return tokens
    }

    const s = str[pos]

    if (state === TYPE) {
      if (s === ' ') {
        consume('TYPE', PARAMS)
        while (shuffleIfNext(' ') && pos < str.length) {}
      } else if (s === ':') {
        consume('TYPE', CONTENT)
        shuffleIfNext(' ')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '[') {
        consume('TYPE', INLINE_CONTENT)
        emit('START_INLINE_CONTENT')
      } else if (s === '(') {
        consume('TYPE', INLINE_PARAMS)
      } else if (s === '\n') {
        consume('TYPE', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === PARAMS) {
      if (s === '[') {
        escapedParamCounter++
        consumingEscapedParams = true
      } else if (consumingEscapedParams && (s === ']')) {
        escapedParamCounter--
        if (escapedParamCounter === 0) {
          consumingEscapedParams = false
        }
      } else if (s === ':' && !consumingEscapedParams) {
        consume('PARAMS', CONTENT)
        shuffleIfNext(' ')
        shuffleIfNext(']')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '\n') {
        escapedParamCounter = 0
        consumingEscapedParams = false
        consume('PARAMS', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === CONTENT) {
      if (s === '@' && !consumingUnparsed) {
        consumeIfNonEmpty('CONTENT', TYPE)
        if (str[pos + 1] === '@') {
          consumingUnparsed = true
          unparsedIndentStart = indent[indent.length - 1]
          pos++
          start++
        }
      } else if (s === '\n') {
        if (consumingSameLineContent) {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
          consumingUnparsed = false
        } else {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
        }
      }
    } else if (state === INLINE_PARAMS) {
      if (s === ')') {
        if (str[pos + 1] === '[') {
          consume('PARAMS', INLINE_CONTENT)
          emit('START_INLINE_CONTENT')
          start++
          pos++
        } else {
          consume('PARAMS', CONTENT)
        }
      }
    } else { // if (state === INLINE_CONTENT) {
      if (s === '\\' && (str[pos + 1] === '[' || str[pos + 1] === ']')) {
        pos++
      } else if (s === '[') {
        escapedInlineCounter++
      } else if (s === ']') {
        if (escapedInlineCounter === 0) {
          consumeIfNonEmpty('CONTENT', CONTENT, ['\\]', '\\['])
          emit('END_INLINE_CONTENT')
          lastInlineContentEndPos = pos + 1
        } else {
          escapedInlineCounter--
        }
      } else if (s === '\n') {
        consume('CONTENT', INLINE_CONTENT)
      }
    }

    if (s === '\n') {
      col = 0
      row++
      newline = true
    } else {
      col++
    }

    pos++
  }

  if (state === TYPE) {
    emit('TYPE', str.substring(start, pos))
  } else if (state === PARAMS) {
    emit('PARAMS', str.substring(start, pos))
  } else if (state === INLINE_PARAMS) {
    err('missing closing ) bracket?')
  } else if (state === INLINE_CONTENT) {
    err('missing closing ] bracket?')
  } else { // if (state === CONTENT) {
    if (start < pos) {
      emit('CONTENT', str.substring(start, pos))
    }
  }

  return tokens
}
274666
public static String capitalize(String text) {
    if (isNullOrEmpty(text)) {
      return text;
    }
    char chars[] = text.toCharArray();
    chars[0] = Character.toUpperCase(chars[0]);
    return new String(chars);
  }