Transforms a bunch of texts into their index representations.

@param texts input texts
@return array of indices of the texts
506266
public Integer[][] textsToSequences(String[] texts) {
        Integer oovTokenIndex  = wordIndex.get(outOfVocabularyToken);
        String[] wordSequence;
        ArrayList<Integer[]> sequences = new ArrayList<>();
        for (String text: texts) {
            if (charLevel) {
                if (lower) {
                    text = text.toLowerCase();
                }
                wordSequence = text.split("");
            } else {
                wordSequence = textToWordSequence(text, filters, lower, split);
            }
            ArrayList<Integer> indexVector = new ArrayList<>();
            for (String word: wordSequence) {
                if (wordIndex.containsKey(word)) {
                    int index = wordIndex.get(word);
                    if (numWords != null && index >= numWords) {
                        if (oovTokenIndex != null)
                            indexVector.add(oovTokenIndex);
                    } else {
                        indexVector.add(index);
                    }
                } else if (oovTokenIndex != null) {
                    indexVector.add(oovTokenIndex);
                }
            }
            Integer[] indices = indexVector.toArray(new Integer[indexVector.size()]);
            sequences.add(indices);
        }
        return sequences.toArray(new Integer[sequences.size()][]);
    }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
1205482
public synchronized List<Sentence> sentences(String text) {
        requireValidHandle();
        Libvoikko lib = getLib();
        List<Sentence> result = new ArrayList<Sentence>();
        if (!isValidInput(text)) {
            result.add(new Sentence(text, SentenceStartType.NONE));
            return result;
        }
        byte[] textBytes = s2n(text);
        int textLen = textBytes.length - 1;
        SizeTByReference sentenceLenByRef = new SizeTByReference();
        while (textLen > 0) {
            int sentenceTypeInt = lib.voikkoNextSentenceStartCstr(handle, textBytes, new SizeT(textLen), sentenceLenByRef);
            int sentenceLen = sentenceLenByRef.getValue().intValue();
            SentenceStartType sentenceType = SentenceStartType.values()[sentenceTypeInt];
            String tokenText = text.substring(0, sentenceLen);
            result.add(new Sentence(tokenText, sentenceType));
            text = text.substring(sentenceLen);
            textBytes = s2n(text);
            textLen = textBytes.length - 1;
        }
        return result;
    }
913911
public List<Hit<V>> parseText(String text)
    {
        int position = 1;
        int currentState = 0;
        List<Hit<V>> collectedEmits = new LinkedList<Hit<V>>();
        for (int i = 0; i < text.length(); ++i)
        {
            currentState = getState(currentState, text.charAt(i));
            storeEmits(position, currentState, collectedEmits);
            ++position;
        }

        return collectedEmits;
    }
877601
public function parse(string $text) : array
    {
        $index = 0;
        $items = Text::split($text, $this->separator);
        $array = [];
        try {
            foreach ($items as $item) {
                $array[] = $this->syntax->parse($item);
                $index += strlen($item) + 1;
            }
        } catch (ParseException $e) {
            $extra = [
                'type' => 'invalid-item',
                'item' => $item,
                'position' => $e->position()
            ];
            throw new ParseException($this, $text, $index + $e->position(),
                "Unable to parse the item '{$item}'", $extra, $e
            );
        }

        return $array;
    }
1041281
public int reverseIndexes(int... indexes) {
        long[] idx = translateToPhysical(ArrayUtil.toLongArray(indexes));
        sort();

        // FIXME: int cast
        return indexesBinarySearch(0, (int) length(), ArrayUtil.toInts(idx));
    }
60439
public final Span[] seqToSpans(final String[] tokens) {
    final Span[] annotatedText = this.sequenceLabeler.tag(tokens);
    final List<Span> probSpans = new ArrayList<Span>(
        Arrays.asList(annotatedText));
    return probSpans.toArray(new Span[probSpans.size()]);
  }
726734
def batch_split_sentences(self, texts: List[str]) -> List[List[str]]:
        
        return [self.split_sentences(text) for text in texts]
17275
public final String transliterate(String text) {
        ReplaceableString result = new ReplaceableString(text);
        transliterate(result);
        return result.toString();
    }
955083
static String[] splitOnTokens(final String text) {
        // used by wildcardMatch
        // package level so a unit test may run on this

        if (text.indexOf('?') == NOT_FOUND && text.indexOf('*') == NOT_FOUND) {
            return new String[] { text };
        }

        final char[] array = text.toCharArray();
        final ArrayList<String> list = new ArrayList<String>();
        final StringBuilder buffer = new StringBuilder();
        char prevChar = 0;
        for (int i = 0; i < array.length; i++) {
            final char ch = array[i];
            if (ch == '?' || ch == '*') {
                if (buffer.length() != 0) {
                    list.add(buffer.toString());
                    buffer.setLength(0);
                }
                if (ch == '?') {
                    list.add("?");
                } else if (prevChar != '*') {// ch == '*' here; check if previous char was '*'
                    list.add("*");
                }
            } else {
                buffer.append(ch);
            }
            prevChar = ch;
        }
        if (buffer.length() != 0) {
            list.add(buffer.toString());
        }

        return list.toArray(new String[list.size()]);
    }
169187
public static String convert(final String text, final List<String> vars) {
		String[] array = new String[vars.size()];
		StringBuilder sb = new StringBuilder();
		convert(sb,text,vars.toArray(array));
		return sb.toString();
	}
173735
public static List<String> splitEqually(String _text, int _len) {
        if (_text == null) {
            return null;
        }
        List<String> ret = new ArrayList<String>((_text.length() + _len - 1) / _len);

        for (int start = 0; start < _text.length(); start += _len) {
            ret.add(_text.substring(start, Math.min(_text.length(), start + _len)));
        }
        return ret;
    }
508317
public String[] split(CharSequence text, int limit)
    {
        try
        {
            InputReader reader = Input.getInstance(text);
            List<String> list = split(reader, limit);
            return list.toArray(new String[list.size()]);
        }
        catch (IOException ex)
        {
            throw new IllegalArgumentException("can't happen", ex);
        }
    }
451224
private String normalizeText(String text) {
        String[] tokens = StringUtils.split(text, "\n");
        for (int i = 0; i < tokens.length; i++) {
            tokens[i] = tokens[i].trim();
        }

        return StringUtils.join(tokens, " ").trim();
    }
802850
public function setTexts(array $texts)
    {
        $this->texts = array_map(function ($text) {
            return preg_replace('/\R/', PHP_EOL, $text);
        }, array_merge($this->texts, $texts));
        return $this;
    }
1221172
public List<String> tokenize(String text) {
		List<String> tokens = new ArrayList<String>();
		text = text.toLowerCase();
		char[] array = text.toCharArray();
		
		//convert all apostrophes to 0x27
		for(int i = 0; i < array.length; i++) {
			if(isApostrofe(array[i])) array[i] = 0x27;
		}
		
		int pos = 0;
		
		//term cycle
		while(pos < array.length) {
			//scan to the start of the term
			while(pos < array.length  && !Character.isLetterOrDigit(array[pos])) pos++;
			int start = pos;
			if(start == array.length) break;
			//scan to the end of the term
			while(pos < array.length  && isLetterOrDigitOrApostrofe(array[pos])) pos++;
			int newpos = pos;
			while(newpos > start && isApostrofe(array[newpos - 1])) newpos--;
			if(newpos > start) tokens.add(new String(array, start, newpos - start));
		}
		return tokens;
	}
738174
public INDArray convert(String text) {
        Tokenizer tokenizer = tokenizerFactory.create(text);
        List<String> tokens = tokenizer.getTokens();
        INDArray create = Nd4j.create(1,wordIndexMap.size());
        Counter<String> tokenizedCounter = new Counter<>();

        for(int i = 0; i < tokens.size(); i++) {
            tokenizedCounter.incrementCount(tokens.get(i),1.0);
        }

        for(int i = 0; i < tokens.size(); i++) {
            if(wordIndexMap.containsKey(tokens.get(i))) {
                int idx = wordIndexMap.get(tokens.get(i));
                int count = (int) tokenizedCounter.getCount(tokens.get(i));
                double weight = tfidfWord(tokens.get(i),count,tokens.size());
                create.putScalar(idx,weight);
            }
        }

        return create;
    }