待匹配字符串的清理空白符和语气助词以及大写数字转化的预处理
760662
private void preHandling(){
		target = stringPreHandlingModule.delKeyword(target, "\\s+"); //清理空白符
		target = stringPreHandlingModule.delKeyword(target, "[的]+"); //清理语气助词
		target = stringPreHandlingModule.numberTranslator(target);//大写数字转化
	}
515604
private void removeLowerCaseBadForUpperCaseGood(AnalyzedSentence input) {
    AnalyzedTokenReadings[] tokens = input.getTokensWithoutWhitespace();
    for (int i = 1; i < tokens.length; i++) {
      if( tokens[i].getReadings().size() > 1
          && StringTools.isCapitalizedWord(tokens[i].getToken())
          && LemmaHelper.hasLemma(tokens[i], Pattern.compile("[А-ЯІЇЄҐ][а-яіїєґ'-].*"), Pattern.compile(".*?:prop")) ) {

        String lowerLemmaToCheck = tokens[i].getAnalyzedToken(0).getLemma().toLowerCase();
        
        List<AnalyzedToken> analyzedTokens = tokens[i].getReadings();
        for(int j=analyzedTokens.size()-1; j>=0; j--) {
          AnalyzedToken analyzedToken = analyzedTokens.get(j);
          
          if( PosTagHelper.hasPosTagPart(analyzedToken, ":bad") 
              && lowerLemmaToCheck.equals(analyzedToken.getLemma()) ) {
            tokens[i].removeReading(analyzedToken);
          }
        }
      }
    }
  }
1317606
def clean(self, html):
        """"""
        return lxml.html.clean.clean_html(unicode(html, self.encoding))
1198991
function getFormat0Segment(glyfUnicodes) {
            var unicodes = [];
            glyfUnicodes.forEach(function (u) {
                if (u.unicode !== undefined && u.unicode < 256) {
                    unicodes.push([u.unicode, u.id]);
                }
            });

            // 按编码排序
            unicodes.sort(function (a, b) {
                return a[0] - b[0];
            });

            return unicodes;
        }
990188
private String normalise(String str) {
		str = str.replaceAll("\\s+", " ");
		str = tc.toSimp(str);
		str = ChineseTrans.toHalfWidth(str);
		return str;
	}
267695
def html_clean(html_code):
    
    temp = re.sub('<script([\s\S]*?)</script>', '', html_code)
    temp = re.sub('<style([\s\S]*?)</style>', '', temp)
    html_cleaned = re.sub('(?is)<.*?>', '', temp)
    for item in html_character:
        html_cleaned = html_cleaned.replace(item, html_character[item])

    log('debug', '网页源代码预处理完成：\n【{}】'.format(html_cleaned))
    return html_cleaned
1010244
function cleanSVG(svgDom, x, y) {
            function getTransformToElement(target, source) {
                var matrix;
                try {
                    matrix = source.getScreenCTM().inverse();
                } catch (e) {
                    throw new Error("Can not inverse source element' ctm.");
                }
                return matrix.multiply(target.getScreenCTM());
            }
            function dealWithPath(d, dealWithPattern) {
                if (!(dealWithPattern instanceof Function)) {
                    dealWithPattern = function() {};
                }
                var strArr = [], pattern = [], cache = [];
                for (var i = 0, l = d.length; i < l; i++) {
                    switch (d[i]) {
                      case "M":
                      case "L":
                      case "T":
                      case "S":
                      case "A":
                      case "C":
                      case "H":
                      case "V":
                      case "Q":
                        {
                            if (cache.length) {
                                pattern.push(cache.join(""));
                                cache = [];
                            }
                            // 脑图的path格式真奇怪...偶尔就给我蹦出来一个"..V123 C..", 那空格几个意思 - -
                            if (pattern[pattern.length - 1] === ",") {
                                pattern.pop();
                            }
                            if (pattern.length) {
                                dealWithPattern(pattern);
                                strArr.push(pattern.join(""));
                                pattern = [];
                            }
                            pattern.push(d[i]);
                            break;
                        }

                      case "Z":
                      case "z":
                        {
                            pattern.push(cache.join(""), d[i]);
                            dealWithPattern(pattern);
                            strArr.push(pattern.join(""));
                            cache = [];
                            pattern = [];
                            break;
                        }

                      case ".":
                      case "e":
                        {
                            cache.push(d[i]);
                            break;
                        }

                      case "-":
                        {
                            if (d[i - 1] !== "e") {
                                if (cache.length) {
                                    pattern.push(cache.join(""), ",");
                                }
                                cache = [];
                            }
                            cache.push("-");
                            break;
                        }

                      case " ":
                      case ",":
                        {
                            if (cache.length) {
                                pattern.push(cache.join(""), ",");
                                cache = [];
                            }
                            break;
                        }

                      default:
                        {
                            if (/\d/.test(d[i])) {
                                cache.push(d[i]);
                            } else {
                                // m a c s q h v l t z情况
                                if (cache.length) {
                                    pattern.push(cache.join(""), d[i]);
                                    cache = [];
                                } else {
                                    // 脑图的path格式真奇怪...偶尔就给我蹦出来一个"..V123 c..", 那空格几个意思 - -
                                    if (pattern[pattern.length - 1] === ",") {
                                        pattern.pop();
                                    }
                                    pattern.push(d[i]);
                                }
                            }
                            if (i + 1 === l) {
                                if (cache.length) {
                                    pattern.push(cache.join(""));
                                }
                                dealWithPattern(pattern);
                                strArr.push(pattern.join(""));
                                cache = null;
                                pattern = null;
                            }
                        }
                    }
                }
                return strArr.join("");
            }
            function replaceWithNode(svgNode, parentX, parentY) {
                if (!svgNode) {
                    return;
                }
                if (svgNode.tagName === "defs") {
                    return;
                }
                if (svgNode.getAttribute("fill") === "transparent") {
                    svgNode.setAttribute("fill", "none");
                }
                if (svgNode.getAttribute("marker-end")) {
                    svgNode.removeAttribute("marker-end");
                }
                parentX = parentX || 0;
                parentY = parentY || 0;
                if (svgNode.getAttribute("transform")) {
                    var ctm = getTransformToElement(svgNode, svgNode.parentElement);
                    parentX -= ctm.e;
                    parentY -= ctm.f;
                    svgNode.removeAttribute("transform");
                }
                switch (svgNode.tagName.toLowerCase()) {
                  case "g":
                    break;

                  case "path":
                    {
                        var d = svgNode.getAttribute("d");
                        if (d) {
                            d = dealWithPath(d, function(pattern) {
                                switch (pattern[0]) {
                                  case "V":
                                    {
                                        pattern[1] = +pattern[1] - parentY;
                                        break;
                                    }

                                  case "H":
                                    {
                                        pattern[1] = +pattern[1] - parentX;
                                        break;
                                    }

                                  case "M":
                                  case "L":
                                  case "T":
                                    {
                                        pattern[1] = +pattern[1] - parentX;
                                        pattern[3] = +pattern[3] - parentY;
                                        break;
                                    }

                                  case "Q":
                                  case "S":
                                    {
                                        pattern[1] = +pattern[1] - parentX;
                                        pattern[3] = +pattern[3] - parentY;
                                        pattern[5] = +pattern[5] - parentX;
                                        pattern[7] = +pattern[7] - parentY;
                                        break;
                                    }

                                  case "A":
                                    {
                                        pattern[11] = +pattern[11] - parentX;
                                        pattern[13] = +pattern[13] - parentY;
                                        break;
                                    }

                                  case "C":
                                    {
                                        pattern[1] = +pattern[1] - parentX;
                                        pattern[3] = +pattern[3] - parentY;
                                        pattern[5] = +pattern[5] - parentX;
                                        pattern[7] = +pattern[7] - parentY;
                                        pattern[9] = +pattern[9] - parentX;
                                        pattern[11] = +pattern[11] - parentY;
                                    }
                                }
                            });
                            svgNode.setAttribute("d", d);
                            svgNode.removeAttribute("transform");
                        }
                        return;
                    }

                  case "image":
                  case "text":
                    {
                        if (parentX && parentY) {
                            var x = +svgNode.getAttribute("x") || 0, y = +svgNode.getAttribute("y") || 0;
                            svgNode.setAttribute("x", x - parentX);
                            svgNode.setAttribute("y", y - parentY);
                        }
                        if (svgNode.getAttribute("dominant-baseline")) {
                            svgNode.removeAttribute("dominant-baseline");
                            svgNode.setAttribute("dy", ".8em");
                        }
                        svgNode.removeAttribute("transform");
                        return;
                    }
                }
                if (svgNode.children) {
                    for (var i = 0, l = svgNode.children.length; i < l; i++) {
                        replaceWithNode(svgNode.children[i], parentX, parentY);
                    }
                }
            }
            svgDom.style.visibility = "hidden";
            replaceWithNode(svgDom, x || 0, y || 0);
            svgDom.style.visibility = "visible";
        }
405543
private static String cleanString(String s) {
		if (s != null) {
			s = s.trim();
			if (s.equals("")) {
				s = null;
			}
		}
		return s;
	}
286092
private static String fixPath(String path) {
		if(StrUtil.isBlank(path)){
			return StrUtil.EMPTY;
		}
		path = StrUtil.cleanBlank(path);							//去除空白符
		path = StrUtil.removeSuffix(path, StrUtil.SLASH); // 去除尾部斜杠
		return path;
	}
1165842
private List<Word>[] fullSeg(String text){
        //文本长度
        final int textLen = text.length();
        //以每一个字作为词的开始，所能切分的词
        final List<String>[] sequence = new LinkedList[textLen];
        if(isParallelSeg()){
            //并行化
            List<Integer> list = new ArrayList<>(textLen);
            for(int i=0; i<textLen; i++){
                list.add(i);
            }
            list.parallelStream().forEach(i->sequence[i] = fullSeg(text, i));
        }else {
            //串行化
            for (int i = 0; i < textLen; i++) {
                sequence[i] = fullSeg(text, i);
            }
        }
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("全切分中间结果：");
            int i=1;
            for(List<String> list : sequence){
                LOGGER.debug("\t"+(i++)+"、"+list);
            }
        }
        //树叶
        List<Node> leaf = new LinkedList<>();
        for(String word : sequence[0]){
            //树根
            Node node = new Node(word);
            //把全切分中间结果（二维数组）转换为合理切分
            buildNode(node, sequence, word.length(), leaf);
        }
        //清理无用数据
        for(int j=0; j<sequence.length; j++){
            sequence[j].clear();
            sequence[j] = null;
        }
        //从所有树叶开始反向遍历出全切分结果
        List<Word>[] res = toWords(leaf);
        leaf.clear();
        return res;
    }
1307575
function clear () {
  var str = '';
  for (var i = 0; i < 20; i++) {
    if (Math.random() < 0.5) {
      str += '\u200B';
    } else {
      str += ' ';
    }
  }
  return str;
}
1220437
function getNewImageDimentions(img, width, height) {
    var detImg = img.width / img.height;
    if (width > 0 && height > 0) {
      // 同时指定了宽高，按原图缩放
      if (width / height > detImg) {
        height = width / detImg;
      } else {
        width = height * detImg;
      }
      return [ width, height ];
    } else if (width > 0) {
      // 只指定宽度的情况
      return [ width, width / detImg ];
    } else if (height > 0) {
      // 只指定高度的情况
      return [ height * detImg, height ];
    } else {
      // 否则原 size 返回
      return [ img.width, img.height ];
    }
  }
980174
function (withWhitespace) {
                var tok;

                // 如果已经有缓存的预读
                if (this.peeked) {
                    if (!withWhitespace && this.peeked.type == lexer.TOKEN_WHITESPACE) {
                        this.peeked = null;
                    } else {
                        tok = this.peeked;
                        this.peeked = null;
                        return tok;
                    }
                }

                tok = this.tokens.nextToken();

                // ... 默认如果下一个tokens是空白字符 则返回下下个token
                // ... 当withWhitespace为true时 即nextToken(true) 则不考虑是否是空白字符
                if (!withWhitespace) {
                    while (tok && tok.type == lexer.TOKEN_WHITESPACE) {
                        tok = this.tokens.nextToken();
                    }
                }

                return tok;
            }
1174661
function clearTagSpaces(html) {

    // 主要移除三种空格
    // 1. 属性名称前面多余的空格
    // 2. 属性值后面多余的空格
    // 3. 等号两边的空格
    // 示例：[   name  =  "value"  ] -> [ name="value" ]
    var result = html.replace(REGEXP_ATTR_SPACE_PRE, function(dirtyAttr, attrName, attrValue) {
        return ' ' + attrName + '=' + attrValue;
    }).replace(REGEXP_ATTR_SPACE_SUF, function(dirtyAttr, attrName, attrValue, a, b, c, sufSpace) {
        var cleanAttr = attrName + '=' + attrValue;

        // 如果结尾没有空格（sufSpace == 0），则不补加单个空字符串
        if (sufSpace.length > 0) {
            cleanAttr += ' ';
        }

        return cleanAttr;
    });

    // 移除标签结束符前面的空格
    // 示例：[<div   >] -> [<div>]
    result = result.replace(/\t+\n/g, '').replace(/\s*>/mg, function(a) {
        return a.replace(/\s+/, '');
    });

    return result;
}
1237142
function otfContours2ttfContours(otfContours) {
            if (!otfContours || !otfContours.length) {
                return otfContours;
            }
            var contours = [];
            for (var i = 0, l = otfContours.length; i < l; i++) {

                // 这里可能由于转换错误导致空轮廓，需要去除
                if (otfContours[i][0]) {
                    contours.push(transformContour(otfContours[i]));
                }
            }

            return contours;
        }
148766
public static String cleanWhitespace(String str) {
        int length = str.length();

        StringBuffer buf = new StringBuffer(length);
        for (int i=0; i<length; i++) {
            char c = str.charAt(i);
            if (Character.isWhitespace(c)) {
                buf.append(' ');
            }
            else {
                buf.append(c);
            }
        }

        return buf.toString();
    }
702746
public static void createImage(String str, Font font, Color backgroundColor, Color fontColor, ImageOutputStream out) throws IORuntimeException {
		// 获取font的样式应用在str上的整个矩形
		Rectangle2D r = font.getStringBounds(str, new FontRenderContext(AffineTransform.getScaleInstance(1, 1), false, false));
		int unitHeight = (int) Math.floor(r.getHeight());// 获取单个字符的高度
		// 获取整个str用了font样式的宽度这里用四舍五入后+1保证宽度绝对能容纳这个字符串作为图片的宽度
		int width = (int) Math.round(r.getWidth()) + 1;
		int height = unitHeight + 3;// 把单个字符的高度+3保证高度绝对能容纳字符串作为图片的高度
		// 创建图片
		BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_BGR);
		Graphics g = image.getGraphics();
		g.setColor(backgroundColor);
		g.fillRect(0, 0, width, height);// 先用背景色填充整张图片,也就是背景
		g.setColor(fontColor);
		g.setFont(font);// 设置画笔字体
		g.drawString(str, 0, font.getSize());// 画出字符串
		g.dispose();
		writePng(image, out);
	}