Tokenize a string and return an iterator over its tokens.
18839
def tokenize(self, string):
        """"""
        it = colorise.compat.ifilter(None, self._pattern.finditer(string))

        try:
            t = colorise.compat.next(it)
        except StopIteration:
            yield string, False
            return

        pos, buf, lm, escapeflag = -1, '', -1, False

        # Check if we need to yield any starting text
        if t.start() > 0:
            yield string[:t.start()], False
            pos = t.start()

        it = itertools.chain([t], it)

        for m in it:
            start = m.start()
            e, s = m.group(2) or '', m.group(3)
            escaped = e.count(self._ESCAPE) % 2 != 0

            if escaped:
                buf += string[pos:m.end(2)-1] + s
                escapeflag = True
            else:
                buf += string[pos:m.start(3)]

                if buf:
                    yield buf, escapeflag
                    buf = ''
                    escapeflag = False

                if lm == start:
                    yield '', False

                yield s, False
                lm = m.end()

            pos = m.end()

        if buf:
            yield buf, escapeflag
            escapeflag = False

        if pos < len(string):
            yield string[pos:], False
715002
def iter_token_lines(tokenlist):
    
    line = []
    for token, c in explode_tokens(tokenlist):
        line.append((token, c))

        if c == '\n':
            yield line
            line = []

    yield line
1065304
def tokenize(text, format=None):
    
    text = Text(text)
    text = text.replace("\t", " ")
    tokens = re.findall(patterns, text)
    tokens = [token[0] for token in tokens]
    if format == "text":
        return " ".join(tokens)
    else:
        return tokens
661874
def tokenize(self, text):
        '''
        
        '''
        start = -1
        tokens = []
        for i, character in enumerate(text):
            if character == ' ' or character == '\t':
                if start >= 0:
                    word = text[start:i]
                    tokens.append({
                        'word': word,
                        'original_text': word,
                        'char_begin': start,
                        'char_end': i})
                    start = -1
            else:
                if start < 0:
                    start = i
        if start >= 0:
            tokens.append({
                'word': text[start:len(text)],
                'original_text': text[start:len(text)],
                'char_begin': start,
                'char_end': len(text)
            })
        return tokens
618346
def _split(self, string):
        
        for i in range(len(string) - self.N + 1):
            yield string[i:i + self.N]
1002952
public static Iterator<Character> stringIterator(final String string) {
        // Ensure the error is found as soon as possible.
        if (string == null)
            throw new NullPointerException();

        return new Iterator<Character>() {
            private int index = 0;

            public boolean hasNext() {
                return index < string.length();
            }

            public Character next() {

                // Throw NoSuchElementException as defined by the Iterator contract,
                // not IndexOutOfBoundsException.
                if (!hasNext())
                    throw new NoSuchElementException();
                return string.charAt(index++);
            }

            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    }
1048963
def tokenize_paragraphs(cls, text):
        """"""
        paragraphs = []
        paragraphs_first_pass = text.split('\n')
        for p in paragraphs_first_pass:
            paragraphs_second_pass = re.split('\s{4,}', p)
            paragraphs += paragraphs_second_pass

        # Remove empty strings from list
        paragraphs = [p for p in paragraphs if p]
        return paragraphs
55738
def tokenize(str)
      raise ArgumentError, 'str must be a String' unless str.is_a?(String)

      j_instance.tokenize(str).to_ary
    end
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
882574
private void process(Iterator<String> tokens) {
        // NOTE: this method is intentionally private to ensure that the
        // IteratorFactory.tokenize() tokenization scheme is enforced on the
        // input data
        long numTokens = 0;
        while (tokens.hasNext()) {
            String token = tokens.next();
            if (doLowerCasing)
                token = token.toLowerCase();
            if (token.matches("[0-9]+"))
                token = "<NUM>";
            if (token.matches("[^\\w\\s;:\\(\\)\\[\\]'!/&?\",\\.<>]"))
                continue;

            Integer count = tokenToCount.get(token);
            tokenToCount.put(token, (count == null) ? 1 : 1 + count);
            numTokens++;
            if (numTokens % UPDATE_INTERVAL == 0 
                    && LOGGER.isLoggable(Level.FINE))
                LOGGER.fine("Processed " + numTokens + " tokens.  Currently " 
                            + tokenToCount.size() + " unique tokens");
        }
    }
681857
def tokenize(stream, separator):
        
        for value in stream:
            for token in value.split(separator):
                if token:
                    yield token.strip()
220085
def tokenize(code):
    """"""
    tok_regex = '|'.join('(?P<{}>{})'.format(*pair) for pair in _tokens)
    tok_regex = re.compile(tok_regex, re.IGNORECASE|re.M)
    line_num = 1
    line_start = 0
    for mo in re.finditer(tok_regex, code):
        kind = mo.lastgroup
        value = mo.group(kind)
        if kind == 'NEWLINE':
            line_start = mo.end()
            line_num += 1
        elif kind == 'SKIP' or value=='':
            pass
        else:
            column = mo.start() - line_start
            yield Token(kind, value, line_num, column)
940881
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}
364563
def tokenize(cls, obj):
    
    tokens = {}

    try:
      token_iterator = cls.make_iterable(obj)
      _lang = cls.language_definition()
      tokens = {k: [] for k in _lang.argument_types}

      prev, current = None, next(token_iterator)

      while True:
        token = [None, None]
        arg_type = None

        for arg_type in _lang.argument_types:
          arg_start_seq = _lang.argument_types[arg_type]
          arg_delimiters = _lang.value_delimiters[arg_type]

          if prev is None and arg_start_seq is None:  # just start to scan and got "default" token
            token[1] = current
            break
          elif arg_start_seq is None and prev[0] is None:  # next default token
            token[1] = current
            break
          elif arg_start_seq is None and prev[0] is not None:  # default token used after non-default tokens
            token[1] = current
            break
          elif arg_start_seq is None:  # value of non-default token
            prev[1] = current
            token = None
            break
          elif current[:len(arg_start_seq)] == arg_start_seq:
            token[0] = current[len(arg_start_seq):]

            for delimiter in arg_delimiters:
              if delimiter == " ":
                continue

              if delimiter in token[0]:
                _delim = str(token[0]).partition(delimiter)
                token = [_delim[0], _delim[2]]
                break
            break

        if token:
          tokens[arg_type].append(token)

        prev, current = token, next(token_iterator)
    except StopIteration:
      return tokens
99250
def __get_tokens(self, row):
        """"""
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            # print("RETURN line", line[0:100])
            yield line
            line = row_tokenizer.next()
855004
def tokenize(self, text, context=0, skip_style_tags=False):
        """"""
        split = self.regex.split(text)
        self._text = [segment for segment in split if segment]
        self._head = self._global = self._depth = 0
        self._bad_routes = set()
        self._skip_style_tags = skip_style_tags

        try:
            tokens = self._parse(context)
        except BadRoute:  # pragma: no cover (untestable/exceptional case)
            raise ParserError("Python tokenizer exited with BadRoute")
        if self._stacks:  # pragma: no cover (untestable/exceptional case)
            err = "Python tokenizer exited with non-empty token stack"
            raise ParserError(err)
        return tokens