Returns pieces of text in element as a List of tokens.

@return A list of all tokens remaining in the underlying Reader
768319
public List<E> tokenize() {
    // System.out.println("tokenize called");
    List<E> result = new ArrayList<E>();
    while (hasNext()) {
      result.add(next());
    }
    return result;
  }
139526
public List<Element> getChildElements() {
		synchronized(lock) {
			if(childElements == null) return Collections.emptyList();
			if(frozen) return childElements;
			return AoCollections.unmodifiableCopyList(childElements);
		}
	}
514366
public synchronized List<Token> tokens(String text) {
        requireValidHandle();
        List<Token> allTokens = new ArrayList<Token>();
        int lastStart = 0;
        for (int i = indexOfSpecialUnknown(text, 0); i != -1; i = indexOfSpecialUnknown(text, i + 1)) {
            allTokens.addAll(tokensNonNull(text.substring(lastStart, i), lastStart));
            allTokens.add(new Token(TokenType.UNKNOWN, Character.toString(text.charAt(i)), i));
            lastStart = i + 1;
        }
        allTokens.addAll(tokensNonNull(text.substring(lastStart), lastStart));
        return allTokens;
    }
372357
@Nullable
  public final AnalyzedTokenReadings[] getUnifiedTokens() {
    if (tokSequence.isEmpty()) {
      return null;
    }
    List<AnalyzedTokenReadings> uTokens = new ArrayList<>();
    for (int j = 0; j < tokSequence.size(); j++) {
      boolean unifiedTokensFound = false; // assume that nothing has been found
      for (int i = 0; i < tokSequenceEquivalences.get(j).size(); i++) {
        int featUnified = 0;
        if (tokSequenceEquivalences.get(j).get(i).containsKey(UNIFY_IGNORE)) {
          addTokenToSequence(uTokens, tokSequence.get(j).getAnalyzedToken(i), j);
          unifiedTokensFound = true;
        } else {
          for (Map.Entry<String, List<String>> feat : unificationFeats.entrySet()) {
            if (tokSequenceEquivalences.get(j).get(i).containsKey(feat.getKey()) &&
                    tokSequenceEquivalences.get(j).get(i).get(feat.getKey()).isEmpty()) {
              featUnified = 0;
            } else {
              featUnified++;
            }
            if (featUnified == unificationFeats.entrySet().size()) {
              addTokenToSequence(uTokens, tokSequence.get(j).getAnalyzedToken(i), j);
              unifiedTokensFound = true;
            }
          }
        }
      }
      if (!unifiedTokensFound) {
        return null;
      }
    }
    return uTokens.toArray(new AnalyzedTokenReadings[0]);
  }
506266
public Integer[][] textsToSequences(String[] texts) {
        Integer oovTokenIndex  = wordIndex.get(outOfVocabularyToken);
        String[] wordSequence;
        ArrayList<Integer[]> sequences = new ArrayList<>();
        for (String text: texts) {
            if (charLevel) {
                if (lower) {
                    text = text.toLowerCase();
                }
                wordSequence = text.split("");
            } else {
                wordSequence = textToWordSequence(text, filters, lower, split);
            }
            ArrayList<Integer> indexVector = new ArrayList<>();
            for (String word: wordSequence) {
                if (wordIndex.containsKey(word)) {
                    int index = wordIndex.get(word);
                    if (numWords != null && index >= numWords) {
                        if (oovTokenIndex != null)
                            indexVector.add(oovTokenIndex);
                    } else {
                        indexVector.add(index);
                    }
                } else if (oovTokenIndex != null) {
                    indexVector.add(oovTokenIndex);
                }
            }
            Integer[] indices = indexVector.toArray(new Integer[indexVector.size()]);
            sequences.add(indices);
        }
        return sequences.toArray(new Integer[sequences.size()][]);
    }
1049202
protected List<EquPart> tokenize() throws Exception
    {
        /*
         * break apart obvious tokens, realizing that some may need further
         * breakage and some may need to be merged back again
         */
        final List<EquPart> tokens = new ArrayList<>();
        Token token = null;
        char c;
        int level = 0;

        for (int a = 0; a < equ.length(); a++)
        {
            c = equ.charAt(a);
            final boolean isWhitespace = Character.isWhitespace(c);

            if (isWhitespace)
                if (!(token != null && token instanceof TokLiteral && token.accepts(c)))
                {
                    /*
                     * spaces are allowed inside of literals
                     */
                    if (token != null)
                        token.addTo(tokens);
                    token = null;
                    continue;
                }

            if (c == '(')
                level++;

            if (c == ')')
                level--;

            if (token != null && token.accepts(c))
                token.put(c);
            else
            {

                if (token != null)
                    token.addTo(tokens);

                token = Token.instanceFor(c);
                token.setLevel(level);
                token.put(c);
            }
        }

        if (token != null)
            token.addTo(tokens);

        return tokens;
    }
43818
private String[] getWords() {

        StringTokenizer tokenizer =
            new StringTokenizer(fSearchWords.getText());
        String[] result = new String[tokenizer.countTokens()];
        int      i      = 0;

        while (tokenizer.hasMoreTokens()) {
            result[i++] = tokenizer.nextToken();
        }    // end of while ((tokenizer.hasMoreTokens()))

        return result;
    }
900248
public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {
                boolean canBreakSentence = true;
                if (t.text.contains("'")) {
                    wordContainsApostrophe(t);
                }
                else if (".".equals(trimmedWord)) {
                    canBreakSentence = wordIsFullStop(t);
                }
                else if (":".equals(trimmedWord)) {
                    wordIsColon(tokens, t);
                }
                else currentSentence.add(t);

                // handling the end of a sentence
                if (canBreakSentence && equalss(trimmedWord, ".", ";", "?", "!")) {
                    paragraph.add(currentSentence);
                    currentSentence = new ArrayList<>();
                }
            }
        }

        if (!currentSentence.isEmpty()) paragraph.add(currentSentence);
        return paragraph;
    }
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
487017
public List<T> getElements() {
		ArrayList<T> list = new ArrayList<>(nodes.size());
		for (Node<T> node : nodes)
			list.add(node.element);
		return list;
	}
	
	/** Return the number of elements. */
	public int size() { return nodes.size(); }
	
}
1373305
public final Span[] multiWordsToSpans(final String[] tokens) {
    final List<Span> multiWordsFound = new LinkedList<Span>();

    for (int offsetFrom = 0; offsetFrom < tokens.length; offsetFrom++) {
      Span multiwordFound = null;
      String tokensSearching[] = new String[] {};

      for (int offsetTo = offsetFrom; offsetTo < tokens.length; offsetTo++) {

        final int lengthSearching = offsetTo - offsetFrom + 1;
        if (lengthSearching > getMaxTokenCount()) {
          break;
        } else {
          tokensSearching = new String[lengthSearching];
          System.arraycopy(tokens, offsetFrom, tokensSearching, 0,
              lengthSearching);

          final String entryForSearch = StringUtils
              .getStringFromTokens(tokensSearching);
          final String entryValue = dictionary
              .get(entryForSearch.toLowerCase());
          if (entryValue != null) {
            multiwordFound = new Span(offsetFrom, offsetTo + 1, entryValue);
          }
        }
      }
      if (multiwordFound != null) {
        multiWordsFound.add(multiwordFound);
        offsetFrom += multiwordFound.length() - 1;
      }
    }
    return multiWordsFound.toArray(new Span[multiWordsFound.size()]);
  }
1198592
public Token lookahead() {
        Token current = token;
        if (current.next == null) {
            current.next = token_source.getNextToken();
        }
        return current.next;
    }
1224981
public @Nonnull String text() {
        return new NodeListSpliterator(node.getChildNodes()).stream()
                        .filter(it -> it instanceof Text)
                        .map(it -> ((Text) it).getNodeValue())
                        .collect(joining());
    }
1206439
public static List<List<Term>> seg2sentence(String text)
    {
        List<List<Term>> sentenceList = SEGMENT.seg2sentence(text);
        for (List<Term> sentence : sentenceList)
        {
            ListIterator<Term> listIterator = sentence.listIterator();
            while (listIterator.hasNext())
            {
                if (!CoreStopWordDictionary.shouldInclude(listIterator.next()))
                {
                    listIterator.remove();
                }
            }
        }

        return sentenceList;
    }
932727
public String[][] getList(final Reader r) throws IOException {
    try {
      setReader(r);
      reset();
      String[][] result = tokenCollection.getList();
      end();
      close();
      return result;
    } catch (MtasParserException e) {
      log.info(e);
      throw new IOException("can't produce list");
    }
  }
741807
def tokens(self):
    
    seq = self.word_tokenizer.transform(Sequence(self.raw))
    return WordList(seq.tokens(), parent=self, language=self.language.code)
323913
private String getSentenceFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String token : tokens) {
      sb.append(token).append(" ");
    }
    final String sentence = sb.toString();
    return sentence;
  }