自动摘要

@param document           目标文档
@param size               需要的关键句的个数
@param sentence_separator 分割目标文档时的句子分割符，正则格式， 如：[。？?！!；;]
@return 关键句列表
508990
public static List<String> extractSummary(String document, int size, String sentence_separator)
    {
        return TextRankSentence.getTopSentenceList(document, size, sentence_separator);
    }
376258
public static Iterable<String> split(String separator, String stringToSplit) {

    String[] StringArray = stringToSplit.split(separator);

    Iterable<String> iterable = Arrays.asList(StringArray);

    return iterable;
  }
1328330
public static List<String> split(String s, String separator) {
        if (s == null) return null;
        List<String> rslt = null;
        if (s.isEmpty() || isEmpty(separator)) {
            rslt = new ArrayList<>(1);
            rslt.add(s);
            return rslt;
        }

        //return s.split(COMMA_SPLIT); - NOT WORKING when compiled to JS

        int start = 0;
        int p = start;
        do {
            int j = s.indexOf(separator, p);
            if (j == -1) {
                if (rslt == null) {
                    rslt = new ArrayList<>(1);
                    rslt.add(s);
                    return rslt;
                }
                rslt.add(s.substring(start));
                break;
            }
            if (j > 0 && s.charAt(j - 1) == '\\') {
                p = j + 1;
                continue;
            } else {
                if (rslt == null) rslt = new ArrayList<String>();
                rslt.add(s.substring(start, j));
                start = j + 1;
                p = start;
            }

        } while (true);

        return rslt;
    }
552533
public List<String> getKeywords(String document, int size)
    {
        return getKeywords(defaultSegment.seg(document), size);
    }
1336938
public int divide(char separator, Text[] parts) {
        int pointer = pos;
        int limit = pos + len;
        int mark = pointer;
        int count = 0;
        
        while (pointer < limit) {
            if (chars[pointer] == separator) {
                parts[count].strain(chars, mark, pointer - mark);
                mark = pointer + 1;
                count++;
                
                if (count == parts.length - 1) {
                    break;
                }
            }
            pointer++;
        }
        
        if(limit > mark)
        {
        	parts[count].strain(chars, mark, limit - mark);
        	count++;
        }
        
        return count;
    }
263710
static List<String> splitSentence(String document, String sentence_separator)
    {
        List<String> sentences = new ArrayList<String>();
        for (String line : document.split("[\r\n]"))
        {
            line = line.trim();
            if (line.length() == 0) continue;
            for (String sent : line.split(sentence_separator))		// [，,。:：“”？?！!；;]
            {
                sent = sent.trim();
                if (sent.length() == 0) continue;
                sentences.add(sent);
            }
        }

        return sentences;
    }
1023144
function split(separator, limit) {
        var O = requireObjectCoercible(this);
        var splitter = separator == undefined ? undefined : separator[SPLIT];
        return splitter !== undefined
          ? splitter.call(separator, O, limit)
          : internalSplit.call(String(O), separator, limit);
      }
379579
public List<WordInfo> discover(BufferedReader reader, int size) throws IOException
    {
        String doc;
        Map<String, WordInfo> word_cands = new TreeMap<String, WordInfo>();
        int totalLength = 0;
        Pattern delimiter = Pattern.compile("[\\s\\d,.<>/?:;'\"\\[\\]{}()\\|~!@#$%^&*\\-_=+，。《》、？：；“”‘’｛｝【】（）…￥！—┄－]+");
        while ((doc = reader.readLine()) != null)
        {
            doc = delimiter.matcher(doc).replaceAll("\0");
            int docLength = doc.length();
            for (int i = 0; i < docLength; ++i)
            {
                int end = Math.min(i + 1 + max_word_len, docLength + 1);
                for (int j = i + 1; j < end; ++j)
                {
                    String word = doc.substring(i, j);
                    if (word.indexOf('\0') >= 0)
                        continue; // 含有分隔符的不认为是词语
                    WordInfo info = word_cands.get(word);
                    if (info == null)
                    {
                        info = new WordInfo(word);
                        word_cands.put(word, info);
                    }
                    info.update(i == 0 ? '\0' : doc.charAt(i - 1), j < docLength ? doc.charAt(j) : '\0');
                }
            }
            totalLength += docLength;
        }

        for (WordInfo info : word_cands.values())
        {
            info.computeProbabilityEntropy(totalLength);
        }
        for (WordInfo info : word_cands.values())
        {
            info.computeAggregation(word_cands);
        }
        // 过滤
        List<WordInfo> wordInfoList = new LinkedList<WordInfo>(word_cands.values());
        ListIterator<WordInfo> listIterator = wordInfoList.listIterator();
        while (listIterator.hasNext())
        {
            WordInfo info = listIterator.next();
            if (info.text.trim().length() < 2 || info.p < min_freq || info.entropy < min_entropy || info.aggregation < min_aggregation
                || (filter && LexiconUtility.getFrequency(info.text) > 0)
                )
            {
                listIterator.remove();
            }
        }
        // 按照频率排序
        MaxHeap<WordInfo> topN = new MaxHeap<WordInfo>(size, new Comparator<WordInfo>()
        {
            public int compare(WordInfo o1, WordInfo o2)
            {
                return Float.compare(o1.p, o2.p);
            }
        });
        topN.addAll(wordInfoList);

        return topN.toList();
    }
675
private static String doNormalizeIgnoreOtherSeparator(String filename, boolean keepSeparator) {
		if (filename == null) {
			return null;
		}
		int size = filename.length();
		if (size == 0) {
			return filename;
		}
		int prefix = 0;
		// int prefix = getPrefixLength(filename);
		// if (prefix < 0) {
		// return null;
		// }

		char[] array = new char[size + 2]; // +1 for possible extra slash, +2
											// for arraycopy
		filename.getChars(0, filename.length(), array, 0);

		// add extra separator on the end to simplify code below
		boolean lastIsDirectory = true;
		if (array[size - 1] != JawrConstant.URL_SEPARATOR_CHAR) {
			array[size++] = JawrConstant.URL_SEPARATOR_CHAR;
			lastIsDirectory = false;
		}

		// adjoining slashes
		for (int i = prefix + 1; i < size; i++) {
			if (array[i] == JawrConstant.URL_SEPARATOR_CHAR && array[i - 1] == JawrConstant.URL_SEPARATOR_CHAR) {
				System.arraycopy(array, i, array, i - 1, size - i);
				size--;
				i--;
			}
		}

		// dot slash
		for (int i = prefix + 1; i < size; i++) {
			if (array[i] == JawrConstant.URL_SEPARATOR_CHAR && array[i - 1] == '.'
					&& (i == prefix + 1 || array[i - 2] == JawrConstant.URL_SEPARATOR_CHAR)) {
				if (i == size - 1) {
					lastIsDirectory = true;
				}
				System.arraycopy(array, i + 1, array, i - 1, size - i);
				size -= 2;
				i--;
			}
		}

		// double dot slash
		outer: for (int i = prefix + 2; i < size; i++) {
			if (array[i] == JawrConstant.URL_SEPARATOR_CHAR && array[i - 1] == '.' && array[i - 2] == '.'
					&& (i == prefix + 2 || array[i - 3] == JawrConstant.URL_SEPARATOR_CHAR)) {
				if (i == prefix + 2) {
					return null;
				}
				if (i == size - 1) {
					lastIsDirectory = true;
				}
				int j;
				for (j = i - 4; j >= prefix; j--) {
					if (array[j] == JawrConstant.URL_SEPARATOR_CHAR) {
						// remove b/../ from a/b/../c
						System.arraycopy(array, i + 1, array, j + 1, size - i);
						size -= (i - j);
						i = j + 1;
						continue outer;
					}
				}
				// remove a/../ from a/../c
				System.arraycopy(array, i + 1, array, prefix, size - i);
				size -= (i + 1 - prefix);
				i = prefix + 1;
			}
		}

		if (size <= 0) { // should never be less than 0
			return "";
		}
		if (size <= prefix) { // should never be less than prefix
			return new String(array, 0, size);
		}
		if (lastIsDirectory && keepSeparator) {
			return new String(array, 0, size); // keep trailing separator
		}
		return new String(array, 0, size - 1); // lose trailing separator
	}
1333524
public static DocumentAndSize [] cut(final Document doc,final PageSize size) {
		NodeList nl = doc.getElementsByTagNameNS("http://www.allcolor.org/xmlns/yahp",
				"pb");
		
		if (nl.getLength() == 0) {
			// see if someone forget namespace decl ?
			nl = doc.getElementsByTagName("yahp:pb");
		}

		// if no pagebreak found, return the given document.
		if (nl.getLength() == 0) {
			return new DocumentAndSize [] { new DocumentAndSize(doc,size) };
		} // end if

		// get the start-end offset of all page breaks.
		PbDocument pbdocs[] = getPbDocs(nl);

		// create as much document as they are pages.
		DocumentAndSize array[] = new DocumentAndSize[pbdocs.length];
		for (int i = 0; i < pbdocs.length; i++) {
			PbDocument pbdoc = pbdocs[i];

			// get start and end offset
			Element pbstart = (Element) pbdoc.getPbStart();
			Element pbend = (Element) pbdoc.getPbEnd();
			
			PageSize tmpSize = getPageSize(pbstart == null ? null : pbstart.getAttribute("size"),size);

			// create a new doc and set the URI
			ADocument ndoc = new CDom2HTMLDocument();
			ndoc.setDocumentURI(doc.getDocumentURI());

			CNamespace xmlnsdef = new CNamespace((pbend != null)
					? pbend.getPrefix()
					: pbstart.getPrefix(),
					"http://www.allcolor.org/xmlns/yahp");
			ndoc.getNamespaceList().add(xmlnsdef);

			if (pbend == null) {
				// from pbstart to the end of the document.
				// create the container node
				Element parentPb = (Element) ndoc.adoptNode(pbstart.getParentNode()
																	   .cloneNode(false));

				// copy all next siblings
				Node Sibling = pbstart.getNextSibling();

				while (Sibling != null) {
					Node node = ndoc.adoptNode(Sibling.cloneNode(true));
					// add to container node
					parentPb.appendChild(node);
					Sibling = Sibling.getNextSibling();
				} // end while

				// copy parent node of the start page break
				Node parent   = pbstart.getParentNode();
				Node nextNode = null;
				Node ppr	  = null;

				while (parent != null) {
					if (parent.getNodeType() == Node.ELEMENT_NODE) {
						// copy the node
						Node node = ndoc.adoptNode(parent.cloneNode(
									false));

						if (nextNode != null) {
							// append the node to the container
							node.appendChild(nextNode);

							if (!("body".equals(ppr.getNodeName()))) {
								// append next sibling of the previous container
								Sibling = ppr.getNextSibling();

								while (Sibling != null) {
									Node n = ndoc.adoptNode(Sibling.cloneNode(
												true));
									node.appendChild(n);
									Sibling = Sibling.getNextSibling();
								} // end while
							} // end if

							nextNode = node;
						} // end if
						else {
							nextNode = parentPb;
						} // end else

						ppr = parent;
					} // end if

					parent = parent.getParentNode();
				} // end while

				ndoc.appendChild(nextNode);
			} // end if
			else {
				Element parentPb   = (Element) ndoc.adoptNode(pbend.getParentNode()
																	   .cloneNode(false));
				Node    Sibling    = pbend.getPreviousSibling();
				boolean hasbeendes = false;

				while (!hasbeendes && (Sibling != null)) {
					if (isDescendant(pbstart, Sibling)) {
						hasbeendes = true;

						break;
					} // end if

					Node node = ndoc.adoptNode(Sibling.cloneNode(true));

					if (parentPb.getChildNodes().getLength() == 0) {
						parentPb.appendChild(node);
					} // end if
					else {
						parentPb.insertBefore(node,
							parentPb.getFirstChild());
					} // end else

					Sibling = Sibling.getPreviousSibling();
				} // end while
				
				if (hasbeendes && pbstart != Sibling) {
					Node c  = ndoc.adoptNode(Sibling.cloneNode(
								true));
					((Element)c).setAttributeNS("http://www.w3.org/2000/xmlns/", "xmlns:yahp", "http://www.allcolor.org/xmlns/yahp");
					Node pb = ((Element) c).getElementsByTagNameNS("http://www.allcolor.org/xmlns/yahp",
							"pb").item(0);
					Node p  = pb;
					Node pp = pb.getParentNode()
									.getPreviousSibling();

					while (p != null) {
						Node tmp = p.getPreviousSibling();
						p.getParentNode()
							 .removeChild(p);
						p = tmp;
					} // end while

					while (pp != null) {
						Node tmp = pp.getPreviousSibling();

						if (tmp == null) {
							Node ppz = pp.getParentNode();

							if (ppz.getPreviousSibling() != null) {
								tmp = ppz.getPreviousSibling();
							} // end if
							else {
								Node prev = null;

								while (prev == null) {
									ppz = ppz.getParentNode();

									if (ppz == null) {
										break;
									} // end if

									prev = ppz.getPreviousSibling();
								} // end while

								if (prev != null) {
									tmp = prev;
								} // end if
							} // end else
						} // end if

						pp.getParentNode()
							  .removeChild(pp);
						pp = tmp;
					} // end while

					if (parentPb.getChildNodes()
								.getLength() == 0) {
						parentPb.appendChild(c);
					} // end if
					else {
						parentPb.insertBefore(c,
								parentPb.getFirstChild());
					} // end else
				} // end if				
				

				Node parent		  = pbend.getParentNode();
				Node previousNode = null;
				Node ppr		  = null;

				while (parent != null) {
					if (parent.getNodeType() == Node.ELEMENT_NODE) {
						if (previousNode != null) {
							Node node = ndoc.adoptNode(parent.cloneNode(
									false));
							node.appendChild(previousNode);

							if (!("body".equals(ppr.getNodeName()))) {
								Sibling = ppr.getPreviousSibling();

								while (!hasbeendes &&
										(Sibling != null)) {
									if (isDescendant(pbstart, Sibling)) {
										// here special handling need to be
										// taken.
										if (pbstart != Sibling) {
											Node c  = ndoc.adoptNode(Sibling.cloneNode(
														true));
											Node pb = ((Element) c).getElementsByTagNameNS("http://www.allcolor.org/xmlns/yahp",
													"pb").item(0);
											Node p  = pb;
											Node pp = pb.getParentNode()
															.getPreviousSibling();

											while (p != null) {
												Node tmp = p.getPreviousSibling();
												p.getParentNode()
													 .removeChild(p);
												p = tmp;
											} // end while

											while (pp != null) {
												Node tmp = pp.getPreviousSibling();

												if (tmp == null) {
													Node ppz = pp.getParentNode();

													if (ppz.getPreviousSibling() != null) {
														tmp = ppz.getPreviousSibling();
													} // end if
													else {
														Node prev = null;

														while (prev == null) {
															ppz = ppz.getParentNode();

															if (ppz == null) {
																break;
															} // end if

															prev = ppz.getPreviousSibling();
														} // end while

														if (prev != null) {
															tmp = prev;
														} // end if
													} // end else
												} // end if

												pp.getParentNode()
													  .removeChild(pp);
												pp = tmp;
											} // end while

											if (node.getChildNodes()
														.getLength() == 0) {
												node.appendChild(c);
											} // end if
											else {
												node.insertBefore(c,
													node.getFirstChild());
											} // end else
										} // end if

										hasbeendes = true;

										break;
									} // end if

									Node n = ndoc.adoptNode(Sibling.cloneNode(
												true));

									if (node.getChildNodes().getLength() == 0) {
										node.appendChild(n);
									} // end if
									else {
										node.insertBefore(n,
											node.getFirstChild());
									} // end else

									Sibling = Sibling.getPreviousSibling();
								} // end while
							} // end if

							previousNode = node;
						} // end if
						else {
							previousNode = parentPb;
						} // end else

						ppr = parent;
					} // end if

					parent = parent.getParentNode();
				} // end while

				ndoc.appendChild(previousNode);
			} // end else

			// copy header
			copyHeader(doc, ndoc);
			array[i] = new DocumentAndSize(ndoc,tmpSize);
		} // end for

		return array;
	}
243717
function parse(size) {
  var matches = size.match(regexp);

  if (!matches)
    return null;

  var multiplicator = matches[3] ? +matches[3].replace(/x$/, '') : 1;

  return {
    width: +matches[1] * multiplicator,
    height: +matches[2] * multiplicator
  };
}
781374
public static List<String> split(String str, String separator, int limit, boolean isTrim, boolean ignoreEmpty, boolean ignoreCase){
		if(StrUtil.isEmpty(str)){
			return new ArrayList<String>(0);
		}
		if(limit == 1){
			return addToList(new ArrayList<String>(1), str, isTrim, ignoreEmpty);
		}
		
		if(StrUtil.isEmpty(separator)){//分隔符为空时按照空白符切分
			return split(str, limit);
		}else if(separator.length() == 1){//分隔符只有一个字符长度时按照单分隔符切分
			return split(str, separator.charAt(0), limit, isTrim, ignoreEmpty, ignoreCase);
		}
		
		final ArrayList<String> list = new ArrayList<>();
		int len = str.length();
		int separatorLen = separator.length();
		int start = 0;
		int i = 0;
		while(i < len){
			i = StrUtil.indexOf(str, separator, start, ignoreCase);
			if(i > -1){
				addToList(list, str.substring(start, i), isTrim, ignoreEmpty);
				start = i + separatorLen;
				
				//检查是否超出范围（最大允许limit-1个，剩下一个留给末尾字符串）
				if(limit > 0 && list.size() > limit-2){
					break;
				}
			}else{
				break;
			}
		}
		return addToList(list, str.substring(start, len), isTrim, ignoreEmpty);
	}
1238577
def simple_separated_format(separator):
    
    return TableFormat(None, None, None, None,
                       headerrow=DataRow('', separator, ''),
                       datarow=DataRow('', separator, ''),
                       padding=0, with_header_hide=None)
1072023
private int getParaPos(String chPara, boolean isParallelThread) {

    if (numParasToCheck == 0 || xComponent == null) {
      return -1;  //  check only the processed paragraph
    }

    if (docCursor == null) {
      docCursor = new DocumentCursorTools(xContext);
    }
    FlatParagraphTools flatPara = null;

    int nParas;
    boolean isReset = false;
    textIsChanged = false;

    if (allParas == null || allParas.size() < 1) {
      if (isParallelThread) {              //  if numThread > 0: Thread may only read allParas
        return -1;
      }
      flatPara = new FlatParagraphTools(xContext);
      if (!resetAllParas(docCursor, flatPara)) {
        return -1;
      }
      if (debugMode > 0) {
        MessageHandler.printToLogFile("+++ resetAllParas (allParas == null): allParas.size: " + allParas.size()
                + ", docID: " + docID + logLineBreak);
      }
      isReset = true;
    }

    // Test if Size of allParas is correct; Reset if not
    nParas = docCursor.getNumberOfAllTextParagraphs();
    if (nParas < 2) {
      return -1;
    } else if (allParas.size() != nParas) {
      if (isParallelThread) {
        return -1;
      }
      if (debugMode > 0) {
        MessageHandler.printToLogFile("*** resetAllParas: allParas.size: " + allParas.size() + ", nParas: " + nParas
                + ", docID: " + docID + logLineBreak);
      }
      List<String> oldParas = allParas;
      if (flatPara == null) {
        flatPara = new FlatParagraphTools(xContext);
      }
      if (!resetAllParas(docCursor, flatPara)) {
        return -1;
      }
      int from = 0;
      while (from < allParas.size() && from < oldParas.size()
          && allParas.get(from).equals(oldParas.get(from))) {
        from++;
      }
      from -= 1 + numParasToCheck;
      resetFrom = from;
      int to = 1;
      while (to <= allParas.size() && to <= oldParas.size()
          && allParas.get(allParas.size() - to).equals(
              oldParas.get(oldParas.size() - to))) {
        to++;
      }
      to = allParas.size() + numParasToCheck - to;
      resetTo = to;
      paragraphsCache.removeAndShift(from, to, allParas.size() - oldParas.size());
      isReset = true;
      if(doResetCheck) {
        from += numParasToCheck;
        to -= numParasToCheck;
        sentencesCache.removeAndShift(from, to, allParas.size() - oldParas.size());
        resetCheck = true;
      }
    }

    // try to get next position from last ViewCursorPosition (proof per dialog box)
    int numLastPara;
    numLastPara = numLastVCPara;
    nParas = findNextParaPos(numLastPara, chPara);
    if (nParas >= 0) {
      numLastVCPara = nParas;
      return nParas;
    }

    // try to get next position from last Position of FlatParagraph (automatic Iteration without Text change)
    numLastPara = numLastFlPara;
    nParas = findNextParaPos(numLastPara, chPara);
    if (nParas >= 0) {
      numLastFlPara = nParas;
      return nParas;
    }

    // try to get ViewCursor position (proof initiated by mouse click)
    nParas = docCursor.getViewCursorParagraph();
    if (nParas >= 0 && nParas < allParas.size() && chPara.equals(allParas.get(nParas))) {
      numLastVCPara = nParas;
      return nParas;
    }

    //  try to get paragraph position from automatic iteration
    if (flatPara == null) {
      flatPara = new FlatParagraphTools(xContext);
    }
    nParas = flatPara.getNumberOfAllFlatPara();

    if (debugMode > 0) {
      MessageHandler.printToLogFile("numLastFlPara: " + numLastFlPara + " (isParallelThread: " + isParallelThread 
          + "); nParas: " + nParas + "; docID: " + docID + logLineBreak);
    }

    if (nParas < allParas.size()) {
      return -1;   //  no automatic iteration
    }
    divNum = nParas - allParas.size();

    nParas = flatPara.getCurNumFlatParagraphs();

    if (nParas < divNum) {
      return -1; //  Proof footnote etc.
    }

    nParas -= divNum;
    numLastFlPara = nParas;
    
    if (!chPara.equals(allParas.get(nParas))) {
      if (isReset || isParallelThread) {
        return -1;
      } else {
        if (debugMode > 0) {
          MessageHandler.printToLogFile("!!! allParas set: NParas: " + nParas + "; divNum: " + divNum
                  + "; docID: " + docID
                  + logLineBreak + "old: " + allParas.get(nParas) + logLineBreak 
                  + "new: " + chPara + logLineBreak);
        }
        allParas.set(nParas, chPara);
        paragraphsCache.remove(nParas);
        if(doResetCheck) {
          sentencesCache.remove(nParas);
          resetCheck = true;
          resetParaNum = nParas;
        }
        resetFrom = nParas - numParasToCheck;
        resetTo = nParas + numParasToCheck + 1;
        textIsChanged = true;
        return nParas;
      }
    }
    if (debugMode > 0) {
      MessageHandler.printToLogFile("nParas from FlatParagraph: " + nParas);
    }
    return nParas;
  }
815238
public static Layout parseDocument(Document document) {
        return new Layout(instance.parseChildren(document, null, Tag.LAYOUT));
    }
421350
public static String joinBySeparator( List<String> items, String separator ) {
        int size = items.size();
        if (size == 0) {
            return "";
        }
        StringBuilder sb = new StringBuilder(items.get(0));
        for( int i = 1; i < size; i++ ) {
            sb.append(separator).append(items.get(i));
        }
        return sb.toString();
    }
1020549
public static List<String> getKeywordList(String document, int size)
    {
        TextRankKeyword textRankKeyword = new TextRankKeyword();

        return textRankKeyword.getKeywords(document, size);
    }