词语标准化

@param word
@return
176458
public static List<Element> wordAlert(String word) {

		char[] chars = WordAlert.alertStr(word);

		List<Element> list = new ArrayList<Element>();

		StringBuilder tempSb = new StringBuilder();

		int status = 0; // 1 num 2 english

		Element element = null;

		for (int i = 0; i < chars.length; i++) {

			if (chars[i] >= '0' && chars[i] <= '9') {
				if (status == 2) {
					element = new Element(Config.getNum(tempSb.toString()));
					element.len = tempSb.length();
					list.add(element);
					tempSb = new StringBuilder();
				}
				tempSb.append(chars[i]);
				status = 1;
			} else if (chars[i] >= 'A' && chars[i] <= 'z') {
				if (status == 1) {
					element = new Element(Config.getEn(tempSb.toString()));
					element.len = tempSb.length();
					list.add(element);
					tempSb = new StringBuilder();
				}
				tempSb.append(chars[i]);
				status = 2;
			} else {
				if (status == 1) {
					element = new Element(Config.getNum(tempSb.toString()));
					element.len = tempSb.length();
					list.add(element);
				} else if (status == 2) {
					element = new Element(Config.getEn(tempSb.toString()));
					element.len = tempSb.length();
					list.add(element);
				}
				tempSb = new StringBuilder();
				list.add(new Element(chars[i]));
				status = 0;
			}

		}

		if (tempSb.length() > 0) {
			if (status == 1) {
				element = new Element(Config.getNum(tempSb.toString()));
				element.len = tempSb.length();
				list.add(element);
			} else if (status == 2) {
				element = new Element(Config.getEn(tempSb.toString()));
				element.len = tempSb.length();
				list.add(element);
			} else {
				LOG.error("err! status :"+status);
			}
		}

		return list;
	}
1342786
public synchronized Vector getVector(String word) {
        try {
            switch (format) {
            case TEXT:
                return new DenseVector(loadTextVector(word));
            case BINARY:
                return new DenseVector(loadBinaryVector(word));
            case SPARSE_TEXT:
                return new CompactSparseVector(loadSparseTextVector(word));
            case SPARSE_BINARY:
                return new CompactSparseVector(loadSparseBinaryVector(word));
            }
        } catch (IOException ioe) {
            // rethrow as something catastrophic must have happened to the
            // underlying .sspace file
            throw new IOError(ioe);
        }
        return null;
    }
1212956
public void addWord(String word) {
		WordTree parent = null;
		WordTree current = this;
		WordTree child;
		char currentChar = 0;
		int length = word.length();
		for(int i = 0; i < length; i++){
			currentChar = word.charAt(i);
			if(false == StopChar.isStopChar(currentChar)){//只处理合法字符
				child = current.get(currentChar);
				if(child == null){
					//无子类，新建一个子节点后存放下一个字符
					child = new WordTree();
					current.put(currentChar, child);
				}
				parent = current;
				current = child;
			}
		}
		if(null != parent){
			parent.setEnd(currentChar);
		}
	}
675877
public String getAmbiguityClass(final String word) {
    final TreeMultimap<Integer, String> mfTagMap = getOrderedMap(word);
    String ambiguityClass = null;
    if (!mfTagMap.isEmpty()) {
      StringJoiner sb = new StringJoiner("-");
      for (Map.Entry<Integer, String> entry : mfTagMap.entries()) {
        sb.add(entry.getValue());
      }
      ambiguityClass = sb.toString();
      //System.err.println(ambiguityClass);
    } else {
      ambiguityClass = "O";
    }
    return ambiguityClass;
  }
90411
protected boolean ignoreWord(String word) throws IOException {
    if (!considerIgnoreWords) {
      return false;
    }
    if (word.endsWith(".") && !wordsToBeIgnored.contains(word)) {
      return isIgnoredNoCase(word.substring(0, word.length()-1));  // e.g. word at end of sentence
    }
    return isIgnoredNoCase(word);
  }
1045164
public Collection<String> wordsNearest(String word, int n) {
        return modelUtils.wordsNearest(word, n);
    }
49350
@Nullable
  public AnalyzedTokenReadings lookup(String word) throws IOException {
    List<AnalyzedTokenReadings> result = tag(Collections.singletonList(word), false);
    AnalyzedTokenReadings atr = result.get(0);
    if (atr.getAnalyzedToken(0).getPOSTag() == null) {
      return null;
    }
    return atr;
  }
416954
@Override
  public List<TaggedWord> tag(String word) {
    List<TaggedWord> lookedUpTerms = mapping.get(word);
    if (lookedUpTerms != null) {
      return Collections.unmodifiableList(lookedUpTerms);
    } else {
      return Collections.emptyList();
    }
  }
1107991
public static List<Word> getWords(String text) {
        List<Word> result = new ArrayList<>();
        List<Word> words = WordParser.parse(text);
        for (Word word : words) {
            result.add(word);
        }
        return result;
    }
1286828
public static TermNatures guessNature(String word) {
		String nature = null;
		SmartForest<String[]> smartForest = SUFFIX_FOREST;
		int len = 0;
		for (int i = word.length() - 1; i >= 0; i--) {
			smartForest = smartForest.get(word.charAt(i));
			if (smartForest == null) {
				break;
			}
			len++;
			if (smartForest.getStatus() == 2) {
				nature = smartForest.getParam()[0];
			} else if (smartForest.getStatus() == 3) {
				nature = smartForest.getParam()[0];
				break;
			}
		}

		if ("nt".equals(nature) && (len > 1 || word.length() > 3)) {
			return TermNatures.NT;
		} else if ("ns".equals(nature)) {
			return TermNatures.NS;
		} else if (word.length() < 5) {
			Result parse = ToAnalysis.parse(word);
			for (Term term : parse.getTerms()) {
				if ("nr".equals(term.getNatureStr())) {
					return TermNatures.NR;
				}
			}
		} else if (DATDictionary.foreign(word)) {
			return TermNatures.NRF;
		}

		return TermNatures.NW;
	}
1030977
public static int findNumberOfSpacesInWord(String word) {
        int count = 0;
        for (int i = 0; i < word.length(); i++)
            if (word.charAt(i) == SPACE_CHAR && (i == 0 || word.charAt(i - 1) != BACK_SLASH))
                count++;

        return count;
    }
503938
static String search(String word) {
		word = word.replaceAll(" ", "+");
		String u;
		if(isUseBaidu)
			u = URL_BAIDU.replace("${wd}", word);
		else
			u = URL_GOOGLE.replace("${wd}", word);
		URL url;
		try {
			url = new URL(u);
		} catch (MalformedURLException e2) {
			// TODO Auto-generated catch block
			e2.printStackTrace();
			return null;
		}

		InputStream istream = null;
		while (true) {
			try {
				HttpURLConnection conn = (HttpURLConnection) url
						.openConnection();
				istream = conn.getInputStream();
				break;
			} catch (IOException e) {
				try {
					Thread.sleep(500);
				} catch (InterruptedException e1) {

				}
			}
		}

		StringBuilder sb = new StringBuilder();
		try {
			BufferedReader reader = new BufferedReader(new InputStreamReader(
					istream, "utf-8"));
			while (true) {
				String buf = reader.readLine();
				if (null == buf)
					break;
				sb.append(buf);				
			}
		} catch (IOException e) {

		}
		String htmlStr =sb.toString();
		//stripping script tags whether the tag contains "/n" or "/r" or not.   
		Matcher m_script = p_script.matcher(htmlStr);   
		htmlStr = m_script.replaceAll("");   

		//stripping style tags whether the tag contains "/n" or "/r" or not.   
		Matcher m_style = p_style.matcher(htmlStr);   
		htmlStr = m_style.replaceAll("");   

		//stripping html tags but continue to have the "/n" and "/r" in right place.   
		Matcher m_html = p_html.matcher(htmlStr);   
		htmlStr = m_html.replaceAll("");   
		
		htmlStr = htmlStr.replaceAll("(&amp;|&nbsp;|&#9654|&copy|◆|&#12288;|&gt;|&lt;)+", " ");
		if(isUseBaidu)
			htmlStr = htmlStr.replaceAll("(百度|_|搜索|\\.\\.\\.|-|2012|2011)", " ");
		htmlStr = htmlStr.replaceAll(String.valueOf(((char)12288))," ");
		htmlStr = htmlStr.replaceAll(" +", " ");
		
		return htmlStr;   

	}
800282
public static boolean isRuleWord(String word) {
        char c = 0;
        for (int i = 0; i < word.length(); i++) {
            c = word.charAt(i);

            if (c != '·') {
                if (c < 256 || filter.contains(c) || (c = WordAlert.CharCover(word.charAt(i))) > 0) {
                    return true;
                }
            }
        }
        return false;
    }
61759
public String tag(final String word, final String posTag) {
    final List<WordData> wdList = this.dictLookup.lookup(word.toLowerCase());
    String newPosTag = null;
    for (final WordData wd : wdList) {
      newPosTag = wd.getTag().toString();
    }
    if (newPosTag == null) {
      newPosTag = posTag;
    }
    return newPosTag;
  }
1233562
protected static String getAorAnArticle(String word) {
		if (Arrays.asList('a', 'e', 'i', 'o', 'u', 'y').contains(Character.toLowerCase(word.charAt(0)))) {
			return "an"; //$NON-NLS-1$
		}
		return "a"; //$NON-NLS-1$
	}
1130172
protected int addWordToVocab(String word)
    {
        vocab[vocabSize] = new VocabWord(word);
        vocabSize++;

        // Reallocate memory if needed
        if (vocabSize + 2 >= vocabMaxSize)
        {
            vocabMaxSize += 1000;
            VocabWord[] temp = new VocabWord[vocabMaxSize];
            System.arraycopy(vocab, 0, temp, 0, vocabSize);
            vocab = temp;
        }
        vocabIndexMap.put(word, vocabSize - 1);
        return vocabSize - 1;
    }
1025975
protected boolean filter(IWord word)
    {
        /*
         * normally word with length less than 2 will
         * be something, well could be ignored 
        */
        if ( word.getValue().length() < 2 ) {
            return false;
        }
        
        //type check
        switch ( word.getType() ) {
            //case IWord.T_BASIC_LATIN:
            case IWord.T_LETTER_NUMBER:
            case IWord.T_OTHER_NUMBER:
            case IWord.T_CJK_PINYIN:
            case IWord.T_PUNCTUATION:
            case IWord.T_UNRECOGNIZE_WORD: {
                return false;
            }
        }
        
        //part of speech check
        String[] poss = word.getPartSpeech();
        if ( poss == null ) return true;
        
        char pos = poss[0].charAt(0);
        switch ( pos ) {
            case 'e': {
                if ( poss[0].equals("en") ) return true;
                return false;
            }
            case 'm': {
                if ( poss[0].equals("mix") ) return true;
                return false;
            }
            case 'q':
            case 'b':
            case 'r':
            case 'z':
            case 'p':
            case 'c':
            case 'u':
            case 'y':
            case 'd':    //@date 2015-11-23
            case 'o':
            case 'h':
            case 'k':
            case 'g':
            case 'x':
            case 'w': {
                return false;
            }
            
            /*case 'n':
            case 'v':
            case 'a':
            case 't':
            case 's':
            case 'f':
            {
                return true;
            }*/
        }
        
        return true;
    }