This method gets as input a string and returns as output a map with the
extracted keywords along with the number of their scores in the text. Their
scores are a combination of occurrences and proximity metrics.

@param text
@return
505342
@Override
    public Map<String, Double> extract(final String text) {
        Map<Integer, String> ID2word = new HashMap<>(); //ID=>Kwd
        Map<Integer, Double> ID2occurrences = new HashMap<>(); //ID=>counts/scores
        Map<Integer, Integer> position2ID = new LinkedHashMap<>(); //word position=>ID maintain the order of insertation
        
        int numberOfWordsInDoc = buildInternalArrays(text, ID2word, ID2occurrences, position2ID);
        
        int maxCombinations = parameters.getMaxCombinations();
        Map<String, Double> keywordsMap = new HashMap<>();
        //move the "window" across the document by 1 word at each time
        for(Map.Entry<Integer, Integer> entry : position2ID.entrySet()) {
            Integer wordID = entry.getValue();
            if(!useThisWord(wordID, ID2word, ID2occurrences)) {
                continue;
            }
            
            Integer position = entry.getKey();
            //Build all the combinations of the current word with other words within the window and estimate the scores
            Map<LinkedList<Integer>, Double> positionCombinationsWithScores = getPositionCombinationsWithinWindow(position, maxCombinations, ID2word, ID2occurrences, position2ID, numberOfWordsInDoc);
            
            //Convert the positions into words and aggregate over their scores.
            for(Map.Entry<LinkedList<Integer>, Double> entry2 : positionCombinationsWithScores.entrySet()) {
                LinkedList<Integer> positionCombination = entry2.getKey();
                
                StringBuilder sb = new StringBuilder(positionCombination.size()*6);
                for(Integer pos : positionCombination) {
                    sb.append(ID2word.get(position2ID.get(pos))).append(" ");
                }
                
                if(sb.length()>0) {
                    String key = sb.toString().trim();
                    double score = entry2.getValue();
                    
                    keywordsMap.put(key, keywordsMap.getOrDefault(key, 0.0)+score);
                }
                
            }
        }
        
        //remove any word that has score less than the min occurrence
        double minScore = parameters.getMinWordOccurrence();
        Iterator<Map.Entry<String, Double>> it = keywordsMap.entrySet().iterator();
        while(it.hasNext()) {
            Map.Entry<String, Double> entry = it.next();
            if(entry.getValue()<minScore) {
                it.remove();
            }
        }
        
        return keywordsMap;
    }
76488
public T lookup(String text)
	{
		if (text == null)
			return null;
		
		for (Entry<String,T> entry :lookupMap.entrySet())
		{
			if (Text.matches(text, entry.getKey()))
				return lookupMap.get(entry.getKey());
			
		}

		return null;

	}
744304
public static List<String> getStringsByRegex(final String text, final String regex) {

    Pattern pattern = Pattern.compile(regex);
    Matcher matcher = pattern.matcher(text);
    List<String> matchedStrings = new ArrayList<String>();
    while (matcher.find()) {
      matchedStrings.add(matcher.group(0));
    }
    return matchedStrings;
  }
1103320
public static final Representation Csv(String text){
        return new ImmutableRep(CONTENT_TYPE_CSV, new ByteArrayInputStream(getBytes(text, MOST_WIDELY_SUPPORTED_ENCODING)));
    }
1347295
public static List<String> extract(String text, int size)
    {
        IPhraseExtractor extractor = new MutualInformationEntropyPhraseExtractor();
        return extractor.extractPhrase(text, size);
    }
1236047
public List<String> keywords(final String source) {
    Map<String, Long> wordCounts = splitChars(source, DEFAULT_THRESHOLD).stream().collect(Collectors.groupingBy(x -> x, Collectors.counting()));
    wordCounts = aggregateKeywords(wordCounts);
    return wordCounts.entrySet().stream().filter(x -> x.getValue() > 1)
      .sorted(Comparator.comparing(x -> -entropy(x.getKey()) * Math.pow(x.getValue(), 0.3)))
      .map(e -> {
        if (isVerbose()) {
          verbose.println(String.format("KEYWORD: \"%s\" - %s * %.3f / %s", e.getKey(), e.getValue(), entropy(e.getKey()), e.getKey().length()));
        }
        return e.getKey();
      }).collect(Collectors.toList());
  }
1100280
private ArrayList<ArrayList<String>> parseString(String text) {

        ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
        StringTokenizer linetoken = new StringTokenizer(text, "\n");
        StringTokenizer token;
        String current;
        while (linetoken.hasMoreTokens()) {
            current = linetoken.nextToken();
            if (current.contains(",")) {
                token = new StringTokenizer(current, ",");
            } else {
                token = new StringTokenizer(current);
            }
            ArrayList<String> line = new ArrayList<String>();
            while (token.hasMoreTokens()) {
                line.add(token.nextToken());
            }
            result.add(line);
        }
        return result;
    }
1063836
public void fromText(String text) {
        String str;
        Occurrence occ;
        List<Term> terms = new ArrayList<Term>();
        StringBuilder sb = new StringBuilder();
        for (List<Term> sentence : seg2sentence(text)) {
            totalTerm += sentence.size();

            // 产生ngram, 总串长度不超过length
            for (int i = 0, j, len = sentence.size(); i < len; ++i) {
                for (j = i; j < len; ++j) {
                    str = sentence.get(j).getRealName();
                    if (termMap.containsKey(str)) {
                        termMap.put(str, termMap.get(str) + 1);
                    } else {
                        addTerm(str);
                    }

                    sb.append(str);
                    if (length < sb.length()) {
                        break;
                    }

                    terms.add(sentence.get(j));

                    str = sb.toString();
                    occ = occurrenceMap.containsKey(str) ? occurrenceMap.get(str) : new Occurrence(new ArrayList<Term>(terms));

                    // 添加左邻项
                    if (0 < i) {
                        occ.addLeftTerm(sentence.get(i - 1).getRealName());
                    }

                    // 添加右临项
                    if (j < len - 1) {
                        occ.addRightTerm(sentence.get(j + 1).getRealName());
                    }

                    // 递增出现频次
                    occ.increaseFrequency();

                    //
                    if (occ.getFrequency() == 1) {
                        addOccurrence(str, occ);
                    }
                }

                //
                terms.clear();
                sb.delete(0, sb.length());
            }
        }
    }
326220
public static List<Term> segment(String text)
    {
        List<Term> termList = new LinkedList<Term>();
        Matcher matcher = WEB_URL.matcher(text);
        int begin = 0;
        int end;
        while (matcher.find())
        {
            end = matcher.start();
            termList.addAll(SEGMENT.seg(text.substring(begin, end)));
            termList.add(new Term(matcher.group(), Nature.xu));
            begin = matcher.end();
        }
        if (begin < text.length()) termList.addAll(SEGMENT.seg(text.substring(begin)));

        return termList;
    }
506266
public Integer[][] textsToSequences(String[] texts) {
        Integer oovTokenIndex  = wordIndex.get(outOfVocabularyToken);
        String[] wordSequence;
        ArrayList<Integer[]> sequences = new ArrayList<>();
        for (String text: texts) {
            if (charLevel) {
                if (lower) {
                    text = text.toLowerCase();
                }
                wordSequence = text.split("");
            } else {
                wordSequence = textToWordSequence(text, filters, lower, split);
            }
            ArrayList<Integer> indexVector = new ArrayList<>();
            for (String word: wordSequence) {
                if (wordIndex.containsKey(word)) {
                    int index = wordIndex.get(word);
                    if (numWords != null && index >= numWords) {
                        if (oovTokenIndex != null)
                            indexVector.add(oovTokenIndex);
                    } else {
                        indexVector.add(index);
                    }
                } else if (oovTokenIndex != null) {
                    indexVector.add(oovTokenIndex);
                }
            }
            Integer[] indices = indexVector.toArray(new Integer[indexVector.size()]);
            sequences.add(indices);
        }
        return sequences.toArray(new Integer[sequences.size()][]);
    }
194499
public static int getLineCount(final String text) {
    final Matcher matcher = LINE_FINDER.matcher(text);
    int i = 0;

    while (matcher.find()) {
      i++;
    }

    return i;
  }
365057
public Category classify(String text) {
		StringTokenizer itr = new StringTokenizer(text);
		Map<Category, Double> scorePerCategory = new HashMap<Category, Double>();
		double bestScore = Double.NEGATIVE_INFINITY;
		Category bestCategory = null;
		while(itr.hasMoreTokens()) {
			String token = NaiveBayesGenerate.normalizeWord(itr.nextToken());
			for(Category category : Category.values()) {
				int count = MapUtils.getInteger(wordCountPerCategory.get(category), token, 0) + 1;
				double wordScore  = Math.log(count / (double) (tokensPerCategory.get(category) + V));
				double totalScore = MapUtils.getDouble(scorePerCategory, category, 0.) + wordScore;
				if(totalScore > bestScore) {
					bestScore = totalScore;
					bestCategory = category;
				}
				scorePerCategory.put(category, totalScore);
			}
		}
		return bestCategory;
	}
905173
public Tag getTagForPattern(CharSequence text) {
		for (final Tag tag : Tag.values()) {
			Pattern pattern = this.compiledPatterns.get(tag);
			if (pattern == null) {
				pattern = Pattern.compile("^\\s*" + getPattern(tag), Pattern.DOTALL); //$NON-NLS-1$
				this.compiledPatterns.put(tag, pattern);
			}
			final Matcher matcher = pattern.matcher(text);
			if (matcher.find()) {
				return tag;
			}
		}
		return null;
	}
499472
public static List<String> asList(String text) {
        if (StringUtils.isBlank(text)) {
            return Collections.emptyList();
        } else {
            try {
                return IOUtils.readLines(new StringReader(text));
            } catch (IOException e) {
                throw new RuntimeException("Cannot get lines", e);
            }
        }
    }
411328
public LanguageProfileBuilder addText(CharSequence text) {
        if (ngramExtractor==null) {
            throw new IllegalStateException("NgramExtractor has not been set yet!");
        }
        for (Map.Entry<String, Integer> entry : ngramExtractor.extractCountedGrams(text).entrySet()) {
            addGram(entry.getKey(), entry.getValue());
        }
        return this;
    }
808045
static String[] splitOnTokens(String text)
    {
        // used by wildcardMatch
        // package level so a unit test may run on this

        if (text.indexOf('?') == -1 && text.indexOf('*') == -1)
        {
            return new String[]
                    {
                        text
                    };
        }

        char[] array = text.toCharArray();
        ArrayList<String> list = new ArrayList<String>();
        StringBuilder buffer = new StringBuilder();
        for (int i = 0; i < array.length; i++)
        {
            if (array[i] == '?' || array[i] == '*')
            {
                if (buffer.length() != 0)
                {
                    list.add(buffer.toString());
                    buffer.setLength(0);
                }
                if (array[i] == '?')
                {
                    list.add("?");
                }
                else if (list.isEmpty()
                        || i > 0 && list.get(list.size() - 1).equals("*") == false)
                {
                    list.add("*");
                }
            }
            else
            {
                buffer.append(array[i]);
            }
        }
        if (buffer.length() != 0)
        {
            list.add(buffer.toString());
        }

        return list.toArray(new String[list.size()]);
    }
244903
@POST
    @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
    @Produces(MediaType.APPLICATION_JSON)
    public static Collection<Utterance> mapText(final String text)
            throws IOException, InterruptedException, JAXBException, SQLException, ParserConfigurationException,
                   SAXException {
        return mapTextWithOptions(new AnalysisRequest(text));
    }