Extract words

@param string $text
@param array $options
- min_char, max_char, case_sensititive, ...
@return array
162000
public function words($text, array $options = []) {
		$text = str_replace([PHP_EOL, "\t"], ' ', $text);

		$pieces = explode(' ', $text);
		$pieces = array_unique($pieces);

		// strip chars like . or ,
		foreach ($pieces as $key => $piece) {
			if (empty($options['case_sensitive'])) {
				$piece = mb_strtolower($piece);
			}
			$search = [',', '.', ';', ':', '#', '', '(', ')', '{', '}', '[', ']', '$', '%', '"', '!', '?', '<', '>', '=', '/'];
			$search = array_merge($search, [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]);
			$piece = str_replace($search, '', $piece);
			$piece = trim($piece);

			if (empty($piece) || !empty($options['min_char']) && mb_strlen($piece) < $options['min_char'] || !empty($options['max_char']) && mb_strlen($piece) > $options['max_char']) {
				unset($pieces[$key]);
			} else {
				$pieces[$key] = $piece;
			}
		}
		$pieces = array_unique($pieces);

		return $pieces;
	}
241589
public static function parseInlineOperator($text)
    {
        $operator_name = 'contains';
        $operator = Arr::get($text, 0, '');
        $value = Arr::get($text, 1, false);

        self::checkInlineOperator($operator, $value);

        if (!empty($operator)) {
            $operator_name = Arr::get(self::getOperator('string', $operator), 'inline', 'contains');
        }

        return [
            $operator_name,
            $operator,
            $value,
        ];
    }
1236920
public function getSnippets($text, $index, $terms, $options = array())
    {
        return $this->sphinx->BuildExcerpts($text, $index, $terms, $options);
    }
1107991
public static List<Word> getWords(String text) {
        List<Word> result = new ArrayList<>();
        List<Word> words = WordParser.parse(text);
        for (Word word : words) {
            result.add(word);
        }
        return result;
    }
488820
private function parseLinks($text)
    {
        $offset = 0;
        $matches = array();
        // Extract all links from the text of the passage (if they exist)
        while (preg_match($this->links_pattern, $text, $matches, 0, $offset)) {
            switch ($matches[2]) {
                case '->':
                    $this->links[] = array('text' => trim($matches[1]), 'link' => trim($matches[3]));
                    $this->raw_links[] = $matches[0];
                    break;
                case '<-':
                case '|':
                    $this->links[] = array('text' => trim($matches[3]), 'link' => trim($matches[1]));
                    $this->raw_links[] = $matches[0];
                    break;
                default:
                    throw new \RuntimeException("Something that looked like a link could not be parsed.");
            }
            $offset = mb_stripos($text, $matches[0]) + mb_strlen($matches[0]);
        }
    }
628613
function splitString( $text )
    {
        $text = self::removeDuplicatedSpaces( trim( self::removeAllQuotes( $text ) ) );

        return empty( $text ) ? array() : explode( " ", $text );
    }
431082
public function getKeyword($text)
    {
        $found = null;
        $text = trim($text);
        $iText = strtoupper(TextNormalizer::stripDiacritics($text));
        foreach ($this->keywords as $keyword) {
            /** @var Keyword $keyword */
            $tmpText = trim($keyword->getText());
            if ($tmpText === $text
                || (
                    empty($keyword->isCaseSensitive()) &&
                    strtoupper(TextNormalizer::stripDiacritics($tmpText)) === $iText)
            ) {
                $found = $keyword;
                break;
            }
        }

        return $found;
    }
8408
public static function __cutAll($sentence, $options = array())
    {
        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $words = array();

        $DAG = self::getDAG($sentence);
        $old_j = -1;

        foreach ($DAG as $k => $L) {
            if (count($L) == 1 && $k > $old_j) {
                $word = mb_substr($sentence, $k, (($L[0]-$k)+1), 'UTF-8');
                array_push($words, $word);
                $old_j = $L[0];
            } else {
                foreach ($L as $j) {
                    if ($j > $k) {
                        $word = mb_substr($sentence, $k, ($j-$k)+1, 'UTF-8');
                        array_push($words, $word);
                        $old_j = $j;
                    }
                }
            }
        }

        return $words;
    }
1094347
public function convertLinks(string $text, array $options = []): string
    {
        $text = $this->replaceProtocol($text);
        $ret  = ' '.$text;

        $attr = '';
        foreach ($options as $key => $value) {
            $attr .= ' '.$key.'="'.$value.'"';
        }

        // Replace Links with http://
        $ret = (string) preg_replace("#(^|[\n ])([\\w]+?://[\\w\\#$%&~/.\\-;:=,?@\\[\\]+]*)#is", '\\1<a href="\\2"'.$attr.'>\\2</a>', $ret);

        // Replace Links without http://
        $ret = (string) preg_replace("#(^|[\n ])((www|ftp)\\.[\\w\\#$%&~/.\\-;:=,?@\\[\\]+]*)#is", '\\1<a href="http://\\2"'.$attr.'>\\2</a>', $ret);

        // Replace Email Addresses
        $ret = (string) preg_replace("#(^|[\n ])([a-z0-9&\\-_.]+?)@([\\w\\-]+\\.([\\w\\-\\.]+\\.)*[\\w]+)#i", '\\1<a href="mailto:\\2@\\3"'.$attr.'>\\2@\\3</a>', $ret);

        return substr($ret, 1);
    }
902303
static function parse_text($text)
    {
        $text = trim($text);
        $result = array();
        $lines = explode(self::$row_sep, $text);
        foreach ($lines as $line)
        {
            $result[] = self::parse_line($line);
        }
        return $result;
    }
862704
public function extractMentions($text)
    {
        $matches = array();

        if (preg_match_all(ComPeopleDomainEntityPerson::PATTERN_MENTION, $text, $matches)) {
            return array_unique($matches[1]);
        } else {
            return array();
        }
    }
678297
public function find_formulas($text) {
        preg_match_all(self::FORMULAS_IN_TEXT_REGEX, $text, $matches);
        return array_combine($matches[1], $matches[1]);
    }
233165
function get_words($text = null)
    {
        $result = array();

        if ($text) {
            // init spellchecker
            $this->init();

            if (!$this->plink) {
                return array();
            }

            // With PSpell we don't need to get suggestions to return misspelled words
            $text = preg_split($this->separator, $text, NULL, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);

            foreach ($text as $w) {
                $word = trim($w[0]);

                // skip exceptions
                if ($this->dictionary->is_exception($word)) {
                    continue;
                }

                if (!pspell_check($this->plink, $word)) {
                    $result[] = $word;
                }
            }

            return $result;
        }

        foreach ($this->matches as $m) {
            $result[] = $m[0];
        }

        return $result;
    }
1037191
public function transform( $text, $options = null )
    {
        if ( null !== $options )
        {
            $this->setOptions( $options );
        }

        $parser = $this->extraEnabled
                ? new MarkdownExtraParser
                : new MarkdownParser;
        $parsed = $parser->transformMarkdown( $text );

        if ( ! empty( $this->linkTarget ) )
        {
            $parsed = preg_replace(
                '#<(a|form)((\s[^>]*)?)>(.*?)</\\1>#',
                '<$1 target="' . htmlspecialchars( $this->linkTarget ) . '"$2>$4</$1>',
                $parsed
            );
        }

        if ( $this->stripScripts )
        {
            $parsed = preg_replace(
                '#<script(\s[^>]*)?>.*?</script>#',
                '',
                $parsed
            );
        }

        return $parsed;
    }
938703
public static function nl2p($text)
    {
        $text = preg_replace('#<br\s*/?>#i', "\n", $text); // replace br with new line

        $paragraphs = '';

        foreach (explode("\n", $text) as $line) {
            if (trim($line)) {
                $paragraphs .= '<p>' . $line . '</p>';
            }
        }

        return $paragraphs;
    }
352711
public function sentimentClassify($text, $options=array()){

        $data = array();
        
        $data['text'] = $text;

        $data = array_merge($data, $options);
        $data = mb_convert_encoding(json_encode($data), 'GBK', 'UTF8');

        return $this->request($this->sentimentClassifyUrl, $data);
    }
877601
public function parse(string $text) : array
    {
        $index = 0;
        $items = Text::split($text, $this->separator);
        $array = [];
        try {
            foreach ($items as $item) {
                $array[] = $this->syntax->parse($item);
                $index += strlen($item) + 1;
            }
        } catch (ParseException $e) {
            $extra = [
                'type' => 'invalid-item',
                'item' => $item,
                'position' => $e->position()
            ];
            throw new ParseException($this, $text, $index + $e->position(),
                "Unable to parse the item '{$item}'", $extra, $e
            );
        }

        return $array;
    }