Split self.text into a list of tokens.

        Args:
            text (str): text to parse

        Yields:
            Token: the tokens generated from the given text.
348574
def lex(self, text):
        
        pos = 0
        while text:
            token_class, match = self.tokens.get_token(text)
            if token_class is not None:
                matched_text = text[match.start():match.end()]
                yield token_class(matched_text)
                text = text[match.end():]
                pos += match.end()
            elif text[0] in self.blank_chars:
                text = text[1:]
                pos += 1
            else:
                raise LexerError(
                        'Invalid character %s in %s' % (text[0], text),
                        position=pos)

        yield self.end_token()
1065191
def tokenizer(text):
    

    stream = deque(text.split("\n"))

    while len(stream) > 0:
        line = stream.popleft()

        if line.startswith("#METABOLOMICS WORKBENCH"):
            yield KeyValue("#METABOLOMICS WORKBENCH", "\n")
            yield KeyValue("HEADER", line)

            for identifier in line.split(" "):
                if ":" in identifier:
                    key, value = identifier.split(":")
                    yield KeyValue(key, value)

        elif line.startswith("#ANALYSIS TYPE"):
            yield KeyValue("HEADER", line)

        elif line.startswith("#SUBJECT_SAMPLE_FACTORS:"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue("#SUBJECT_SAMPLE_FACTORS", "\n")

        elif line.startswith("#"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue(line.strip(), "\n")

        elif line.startswith("SUBJECT_SAMPLE_FACTORS"):
            key, subject_type, local_sample_id, factors, additional_sample_data = line.split("\t")
            # factors = [dict([[i.strip() for i in f.split(":")]]) for f in factors.split("|")]
            yield SubjectSampleFactors(key.strip(), subject_type, local_sample_id, factors, additional_sample_data)

        elif line.endswith("_START"):
            yield KeyValue(line, "\n")

            while not line.endswith("_END"):
                line = stream.popleft()
                if line.endswith("_END"):
                    yield KeyValue(line.strip(), "\n")
                else:
                    data = line.split("\t")
                    yield KeyValue(data[0], tuple(data))

        else:
            if line:
                if line.startswith("MS:MS_RESULTS_FILE") or line.startswith("NM:NMR_RESULTS_FILE"):
                    try:
                        key, value, extra = line.split("\t")
                        extra_key, extra_value = extra.strip().split(":")
                        yield KeyValueExtra(key.strip()[3:], value, extra_key, extra_value)
                    except ValueError:
                        key, value = line.split("\t")
                        yield KeyValue(key.strip()[3:], value)
                else:
                    try:
                        key, value = line.split("\t")
                        if ":" in key:
                            if key.startswith("MS_METABOLITE_DATA:UNITS"):
                                yield KeyValue(key.strip(), value)
                            else:
                                yield KeyValue(key.strip()[3:], value)
                        else:
                            yield KeyValue(key.strip(), value)
                    except ValueError:
                        print("LINE WITH ERROR:\n\t", repr(line))
                        raise

    yield KeyValue("#ENDSECTION", "\n")
    yield KeyValue("!#ENDFILE", "\n")
399299
def tokenize(self, text: str, model: object = None):
        
        sentences = re.split(self.pattern, text)
        return sentences
953804
def tokenize(text):

    

    stem = PorterStemmer().stem
    tokens = re.finditer('[a-z]+', text.lower())

    for offset, match in enumerate(tokens):

        # Get the raw token.
        unstemmed = match.group(0)

        yield { # Emit the token.
            'stemmed':      stem(unstemmed),
            'unstemmed':    unstemmed,
            'offset':       offset
        }
701036
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._tokenizer.tokenize_to_spacy_doc(text)
        self._load_matcher()

        matches = [x for x in self._matcher(doc) if x[1] != x[2]]
        pos_filtered_matches = []
        neg_filtered_matches = []
        for idx, start, end in matches:
            span_doc = self._tokenizer.tokenize_to_spacy_doc(doc[start:end].text)
            this_spacy_rule = self._matcher.get(idx)
            relations = self._find_relation(span_doc, this_spacy_rule)
            rule_id, _ = self._hash_map[idx]
            this_rule = self._rule_lst[rule_id]
            if self._filter_match(doc[start:end], relations, this_rule.patterns):
                value = self._form_output(doc[start:end], this_rule.output_format, relations, this_rule.patterns)
                if this_rule.polarity:
                    pos_filtered_matches.append((start, end, value, rule_id, relations))
                else:
                    neg_filtered_matches.append((start, end, value, rule_id, relations))

        return_lst = []
        if pos_filtered_matches:
            longest_lst_pos = self._get_longest(pos_filtered_matches)
            if neg_filtered_matches:
                longest_lst_neg = self._get_longest(neg_filtered_matches)
                return_lst = self._reject_neg(longest_lst_pos, longest_lst_neg)
            else:
                return_lst = longest_lst_pos

        extractions = []
        for (start, end, value, rule_id, relation) in return_lst:
            this_extraction = Extraction(value=value,
                                         extractor_name=self.name,
                                         start_token=start,
                                         end_token=end,
                                         start_char=doc[start].idx,
                                         end_char=doc[end-1].idx+len(doc[end-1]),
                                         rule_id=rule_id.split("rule_id##")[0],
                                         match_mapping=relation)
            extractions.append(this_extraction)

        return extractions
1178426
def run(self, text):
    '''
    '''

    stack = ['root']
    pos = 0

    patterns = self.tokens[stack[-1]]

    while True:
      for pat, action, new_state in patterns:
        m = pat.match(text, pos)
        if m:
          if action:
            #print('## MATCH: {} -> {}'.format(m.group(), action))
            yield (pos, m.end()-1), action, m.groups()

          pos = m.end()

          if new_state:
            if isinstance(new_state, int): # Pop states
              del stack[new_state:]
            else:
              stack.append(new_state)

            #print('## CHANGE STATE:', pos, new_state, stack)
            patterns = self.tokens[stack[-1]]

          break

      else:
        try:
          if text[pos] == '\n':
            pos += 1
            continue
          pos += 1
        except IndexError:
          break
546232
def tree_words(node):
    
    for word in split_text(tree_text(node)):
        word = word.strip()
        if word:
            yield word
860136
def lex(self, text, start=0):
        
        max = len(text)
        eaten = start
        s = self.state
        r = self.regexes
        toks = self.toks
        while eaten < max:
            for match in r[s].finditer(text, eaten):
                name = match.lastgroup
                tok = toks[name]
                toktext = match.group(name)
                eaten += len(toktext)
                yield (tok.name, toktext)

                if tok.next:
                    s = tok.next
                    break

        self.state = s
634973
def extract_from_text(text):
    
    return tools.remove_duplicates([i[0]
                                    for i in REGEX.findall(text) if i != ''])
1302431
def parse(self, text):
        
        #: Do not process empty strings (Issue #3)
        if text.strip() == "":
            return ""
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif text.strip() in PUNCTUATION:
            _sym = text.strip()
            if _sym in tuple('.?!'):
                _tag = "."
            else:
                _tag = _sym
            if self.lemmata:
                return "{0}/{1}/O/O/{0}".format(_sym, _tag)
            else:
                return "{0}/{1}/O/O".format(_sym, _tag)
        if self.tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(text))
        else:
            _tokenized = text

        _parsed = pattern_parse(_tokenized,
                                # text is tokenized before it is passed on to
                                # pattern.de.parse
                                tokenize=False,
                                tags=self.tags, chunks=self.chunks,
                                relations=self.relations, lemmata=self.lemmata,
                                encoding=self.encoding, tagset=self.tagset)
        if self.pprint:
            _parsed = pattern_pprint(_parsed)

        return _parsed
314848
def word_tokenize(text, stopwords=_stopwords, ngrams=None, min_length=0, ignore_numeric=True):
    
    if ngrams is None:
        ngrams = 1

    text = re.sub(re.compile('\'s'), '', text)  # Simple heuristic
    text = re.sub(_re_punctuation, '', text)

    matched_tokens = re.findall(_re_token, text.lower())
    for tokens in get_ngrams(matched_tokens, ngrams):
        for i in range(len(tokens)):
            tokens[i] = tokens[i].strip(punctuation)

            if len(tokens[i]) < min_length or tokens[i] in stopwords:
                break
            if ignore_numeric and isnumeric(tokens[i]):
                break
        else:
            yield tuple(tokens)
1205305
def split_token_in_parts(token):
    
    result = []
    current = []
    for part in token + (':', ):
        if part == ':':
            if current:
                result.append(tuple(current))
                current = []
        else:
            current.append(part)

    return result
1221172
public List<String> tokenize(String text) {
		List<String> tokens = new ArrayList<String>();
		text = text.toLowerCase();
		char[] array = text.toCharArray();
		
		//convert all apostrophes to 0x27
		for(int i = 0; i < array.length; i++) {
			if(isApostrofe(array[i])) array[i] = 0x27;
		}
		
		int pos = 0;
		
		//term cycle
		while(pos < array.length) {
			//scan to the start of the term
			while(pos < array.length  && !Character.isLetterOrDigit(array[pos])) pos++;
			int start = pos;
			if(start == array.length) break;
			//scan to the end of the term
			while(pos < array.length  && isLetterOrDigitOrApostrofe(array[pos])) pos++;
			int newpos = pos;
			while(newpos > start && isApostrofe(array[newpos - 1])) newpos--;
			if(newpos > start) tokens.add(new String(array, start, newpos - start));
		}
		return tokens;
	}
514366
public synchronized List<Token> tokens(String text) {
        requireValidHandle();
        List<Token> allTokens = new ArrayList<Token>();
        int lastStart = 0;
        for (int i = indexOfSpecialUnknown(text, 0); i != -1; i = indexOfSpecialUnknown(text, i + 1)) {
            allTokens.addAll(tokensNonNull(text.substring(lastStart, i), lastStart));
            allTokens.add(new Token(TokenType.UNKNOWN, Character.toString(text.charAt(i)), i));
            lastStart = i + 1;
        }
        allTokens.addAll(tokensNonNull(text.substring(lastStart), lastStart));
        return allTokens;
    }
669999
def split_sentences(text):
    
    sentence_delimiters = re.compile(u'[\\[\\]\n.!?,;:\t\\-\\"\\(\\)\\\'\u2019\u2013]')
    sentences = sentence_delimiters.split(text)
    return sentences
1358946
def tokenize_words(self, text):
        """"""
        return [
            self.strip_punctuation(word) for word in text.split(' ')
            if self.strip_punctuation(word)
        ]
1102062
def split_sentences_regex(text):
    
    parts = regex.split(r'([a-zA-Z0-9][.?!])[\s$]', text)
    sentences = [''.join(s) for s in zip(parts[0::2], parts[1::2])]
    return sentences + [parts[-1]] if len(parts) % 2 else sentences