Extract tags and entities from a given URL. We download the HTML from the URL, and submit
that to Calais using the extractFromText function

@param url The URL to analyze.
@param cb The callback function, of form function(result, error)
344639
function(url, cb) {
        var calais = this;

        if (!calais.validateOptions())
            return cb({}, 'Bad options');

        //Make sure we were given a URL.
        if (this._undefinedOrNull(url) || typeof url != 'string' || url.length == 0)
            return cb({}, 'No URL given.');

        //Make sure it's a valid URL.
        if (!(/^(https?|ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(\#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$/i.test(url)))
            return cb({}, 'Bad URL');


        request(url, function(error, response, html) {
            if (error)
                return cb({}, error);

            //Limts to just under 100kb....
            html = html.substring(0, 95000);
            
            //We can upload the html directly to Calais if we set the contentType as text/html
            var params = {
                'Host'                   : calais.options.apiHost,
                'x-ag-access-token'      : calais.apiKey,
                'x-calais-language'      : calais.options.language,
                'Content-Type'           : 'text/html',
                'Accept'                 : 'application/json',
                'Content-Length'         : html.length,
                'OutputFormat'           : 'application/json'
            };

            var options = {
                uri    : 'https://' + calais.options.apiHost + calais.options.apiPath,
                method : 'POST',
                body   : html,
                headers: params
            };


            request(options, function(error, response, calaisData) {

                if (error)
                    return cb({}, error);

                if (response === undefined) {
                    return cb({}, 'Undefined Calais response');
                } else if (response.statusCode === 200) {
                    if (!calaisData)
                        return cb({}, calais.errors);

                    try {
                        // parse to a Javascript object if requested
                        var result = JSON.parse(calaisData);
                        result = (typeof result === 'string') ? JSON.parse(result) : result;

                        var parsedResult;
                        if (calais.options.parseData) {
                            parsedResult = calais._parseCalaisData(result, calais.options.minConfidence);
                        } else {
                            parsedResult = result;
                        }

                        return cb(parsedResult, calais.errors);
                    } catch(e) {
                        return cb({}, e);
                    }

                } else
                    return cb({}, 'Request error: ' + (typeof response === 'string' ? response : JSON.stringify(response)));


            });

        });

    }
1283501
public Html download(String url, String charset) {
        Page page = download(new Request(url), Site.me().setCharset(charset).toTask());
        return (Html) page.getHtml();
    }
507841
public void setHref(String url) {
        if (url == null)
            throw new RuntimeException("Cannot set URL to null. Call removeUrl() if you wanted to remove the URL");
        if (anchor == null) {
            if (controlGroupRoot != null) {
                //!!! following code is semi-working, it's not reconstructed item correctly
                remove(controlGroupRoot);
                anchor = Document.get().createAnchorElement();
                anchor.setAttribute("href", url);
                moveThisChildrenToAnchor();
                cleanUpLI();
                prepareAnchorForControlGroup();
                getElement().appendChild(anchor);
                checkAnchorPanel();
            } else {
                // need to make anchor and move children to it
                anchor = Document.get().createAnchorElement();
                moveThisChildrenToAnchor();
                getElement().appendChild(anchor);
            }
            addItemActivationHandlers();
        }
        anchor.setAttribute("href", url);
    }
289152
function (url, method, cb) {

            request({
                url: url,
                method: method
            }, function (error, response, body) {

                if (error) {
                    return cb(error);
                }

                if (response.statusCode == 200 || response.statusCode == 201) {

                    return cb(null, body);

                } else {

                    return cb(new Error('Server responded with error: ' + response.statusCode));

                }

            })

        }
548557
public static org.jsoup.nodes.Document getDocumentFromUrl(String url) throws IOException {
        return Jsoup.connect(url).get();
    }
1046110
public String getPageHtml(String url) {
        HttpClient httpClient = new HttpClient();
        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(2000);
        httpClient.getHttpConnectionManager().getParams().setSoTimeout(3000);
        GetMethod getMethod = new GetMethod(url);
        try {
            int statusCode = httpClient.executeMethod(getMethod);
            if (statusCode != HttpStatus.SC_OK) {
                LOG.warn("HttpStatus not equal to 200, some problems may occurred:"
                        + getMethod.getStatusLine());
            }
            InputStream inputStream = getMethod.getResponseBodyAsStream();
            BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
            StringBuffer stringBuffer = new StringBuffer();
            String html = "";
            while ((html = br.readLine()) != null) {
                stringBuffer.append(html);
            }
            return stringBuffer.toString();
        } catch (Exception e) {
            LOG.error("Error to get html of url=" + url + " due to " + e.getMessage());
            throw new RuntimeException("Get html failed, " + e.getMessage(), e);
        } finally {
            try {
                getMethod.releaseConnection();
            } catch (Exception e2) {
                // TODO: handle exception
            }
        }
    }
671122
public static synchronized Document getEventDocFromUrl(String url) throws TvDbException {
        Document doc = null;

        String webPage = getValidWebpage(url);
        
        // If the webpage returned is null or empty, quit
        if(StringUtils.isBlank(webPage)){
            return null;
        }
        
        try (InputStream in = new ByteArrayInputStream(webPage.getBytes(CHARSET))) {
            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
            DocumentBuilder db = dbf.newDocumentBuilder();
            doc = db.parse(in);
            doc.getDocumentElement().normalize();
        } catch (UnsupportedEncodingException ex) {
            throw new TvDbException(ApiExceptionType.INVALID_URL, "Unable to encode URL", url, ex);
        } catch (ParserConfigurationException | SAXException | IOException error) {
            throw new TvDbException(ApiExceptionType.MAPPING_FAILED, ERROR_UNABLE_TO_PARSE, url, error);
        }

        return doc;
    }
227296
protected Document getHttpDocument(String url)
			throws IOException, SAXException {
		URL u = new URL(url);
		URLConnection conn = u.openConnection();
		conn.setConnectTimeout(connectTimeout);
		conn.setReadTimeout(readTimeout);
		return (getDocumentBuilder()).parse(conn.getInputStream(),url);
	}
929701
private SortedSet<String> requestRepresentativeDomains(String url) {
		try {

			//System.out.println(url);

			final SortedSet<String> results = new TreeSet<String>();
			DefaultHandler handler = new DefaultHandler() {
				@Override
				public void startElement(String uri, String localName,String qName,
						Attributes attributes) throws SAXException {

					//System.out.println("Start Element :" + qName);

					if (qName.equalsIgnoreCase("representative")) {
						String name = attributes.getValue("name");
						results.add(name);
					}
				}
			};
			handleRestRequest(url,handler);
			return results;
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (SAXException e) {
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		}
		return null;
	}
458581
@Override
    public String extractTokenFromContent(Context context) {
        // check query String
        String token = context.request().parameter(getTokenName());

        if (token == null) {
            // Check a specified header
            token = context.header(CSRF_TOKEN_HEADER);
        }

        // If still not found, check in body
        if (token == null) {
            if (context.request().contentType().startsWith(MimeTypes.FORM)
                    || context.request().contentType().startsWith(MimeTypes.MULTIPART)) {
                List<String> list = context.form().get(getTokenName());
                if (list != null && !list.isEmpty()) {
                    return list.get(0);
                }
            }
        }
        return token;
    }
1164310
@Override
    public int doEndTag() throws JspException {

        ServletRequest req = pageContext.getRequest();
        A_CmsConfiguredHtmlParser parser;
        if (m_allowInlineEdit && CmsJspTagEditable.isEditableRequest(req)) {
            // during inline editing the content should not be parsed
            try {
                getBodyContent().writeOut(pageContext.getOut());
                release();
            } catch (Exception ex) {
                release();
                if (LOG.isErrorEnabled()) {
                    LOG.error(Messages.get().getBundle().key(Messages.ERR_PROCESS_TAG_1, TAG_NAME), ex);
                }
                // this is severe
                throw new JspException(ex);
            }
        } else {
            // This will always be true if the page is called through OpenCms
            if (CmsFlexController.isCmsRequest(req)) {
                String content = "";
                try {
                    if (CmsStringUtil.isEmpty(m_configuredParserClassname)) {
                        if (LOG.isErrorEnabled()) {
                            LOG.error(
                                Messages.get().getBundle().key(
                                    Messages.GUI_ERR_TAG_ATTRIBUTE_MISSING_2,
                                    new Object[] {TAG_NAME, ATT_VISITOR_CLASS}));
                        }

                    }
                    // wrong attribute visitorClass -> content will remain empty, but no exception is
                    // thrown
                    try {
                        // load
                        Class<?> cl = Class.forName(m_configuredParserClassname);
                        // Instantiate
                        Object instance = cl.newInstance();
                        // cast
                        parser = (A_CmsConfiguredHtmlParser)instance;
                        parser.setParam(m_param);
                        // cms object:
                        CmsFlexController controller = CmsFlexController.getController(req);
                        CmsObject cms = controller.getCmsObject();
                        parser.setCmsObject(cms);
                        content = parseTagAction(getBodyContent().getString(), pageContext, parser);

                    } catch (Exception e) {
                        if (LOG.isErrorEnabled()) {
                            LOG.error(
                                Messages.get().getBundle().key(
                                    Messages.GUI_ERR_TAG_ATTRIBUTE_INVALID_3,
                                    new Object[] {
                                        TAG_NAME,
                                        ATT_VISITOR_CLASS,
                                        A_CmsConfiguredHtmlParser.class.getName()}),
                                e);
                        }
                        e.printStackTrace(System.err);
                    }

                } finally {
                    try {
                        getBodyContent().clear();
                        getBodyContent().print(content);
                        getBodyContent().writeOut(pageContext.getOut());
                        release();

                    } catch (Exception ex) {
                        release();
                        if (LOG.isErrorEnabled()) {
                            LOG.error(Messages.get().getBundle().key(Messages.ERR_PROCESS_TAG_1, TAG_NAME), ex);
                        }
                        // this is severe
                        throw new JspException(ex);
                    }

                }

            }
        }
        return EVAL_PAGE;
    }
257060
private ExtractResult extractHtmlTemplate(String url, HtmlTemplate htmlTemplate, Document doc) {
        //一个页面模板对应一个抽取结果
        ExtractResult extractResult = new ExtractResult();
        extractResult.setUrl(url);
        extractResult.setTableName(htmlTemplate.getTableName());
        List<CssPath> cssPaths = htmlTemplate.getCssPaths();
	    //页面模板中定义的所有CSS路径和抽取表达式全部抽取成功，才算抽取成功
        //只要有一个CSS路径或抽取表达式失败，就是抽取失败
        for (CssPath cssPath : cssPaths) {
            // 抽取一条CSS PATH
            Elements elements = doc.select(cssPath.getCssPath());
            // 如果CSS路径匹配多个元素，则抽取字段为多值
            for (Element element : elements) {
                String text = null;
                if(StringUtils.isBlank(cssPath.getAttr())){
                    //提取文本
                    text = element.text();
                }else{
                    //提取属性
                    text = element.attr(cssPath.getAttr());
                }
                if (StringUtils.isNotBlank(text)) {
                    // 成功提取文本
                    if (cssPath.hasExtractFunction()) {
                        //使用CSS路径下的抽取函数做进一步抽取
                        for (ExtractFunction pf : cssPath.getExtractFunctions()) {
                            text = ExtractFunctionExecutor.execute(text, doc, cssPath, pf.getExtractExpression());
                            if (text != null) {
                                ExtractResultItem extractResultItem = new ExtractResultItem();
                                extractResultItem.setField(pf.getFieldName());
                                extractResultItem.setValue(text);
                                extractResult.addExtractResultItem(extractResultItem);
                            } else {
                                ExtractFailLog extractFailLog = new ExtractFailLog();
                                extractFailLog.setUrl(url);
                                extractFailLog.setUrlPattern(htmlTemplate.getUrlPattern().getUrlPattern());
                                extractFailLog.setTemplateName(htmlTemplate.getTemplateName());
                                extractFailLog.setCssPath(cssPath.getCssPath());
                                extractFailLog.setExtractExpression(pf.getExtractExpression());
                                extractFailLog.setTableName(htmlTemplate.getTableName());
                                extractFailLog.setFieldName(pf.getFieldName());
                                extractFailLog.setFieldDescription(pf.getFieldDescription());
                                extractResult.addExtractFailLog(extractFailLog);
                                //未抽取到结果，保存抽取失败日志并停止抽取，抽取失败
                                //快速失败模式
                                //如果要记录所有失败日志，则去除下面一行返回的代码
                                return extractResult;
                            }
                        }
                    } else {
                        //使用CSS路径抽取的结果
                        ExtractResultItem extractResultItem = new ExtractResultItem();
                        extractResultItem.setField(cssPath.getFieldName());
                        extractResultItem.setValue(text);
                        extractResult.addExtractResultItem(extractResultItem);
                    }
                } else {
                    //未抽取到结果，保存抽取失败日志并停止抽取，抽取失败
                    ExtractFailLog extractFailLog = new ExtractFailLog();
                    extractFailLog.setUrl(url);
                    extractFailLog.setUrlPattern(htmlTemplate.getUrlPattern().getUrlPattern());
                    extractFailLog.setTemplateName(htmlTemplate.getTemplateName());
                    extractFailLog.setCssPath(cssPath.getCssPath());
                    extractFailLog.setExtractExpression("");
                    extractFailLog.setTableName(htmlTemplate.getTableName());
                    extractFailLog.setFieldName(cssPath.getFieldName());
                    extractFailLog.setFieldDescription(cssPath.getFieldDescription());
                    extractResult.addExtractFailLog(extractFailLog);
                    //未抽取到结果，保存抽取失败日志并停止抽取，抽取失败
                    //快速失败模式
                    //如果要记录所有失败日志，则去除下面一行返回的代码
                    return extractResult;
                }
            }
        }
        return extractResult;
    }
771898
function(url, dest, cb) {
      var get = request(url);
      get.on('response',  function (res) {
        res.pipe(fs.createWriteStream(dest));
        res.on('end', function () {
          cb();
        });
        res.on('error', function (err) {
          cb(err);
        });
      });
    }
219038
public boolean acceptsURL(String url) {

        if (url == null) {
            return false;
        }

        return url.regionMatches(true, 0, DatabaseURL.S_URL_PREFIX, 0,
                                 DatabaseURL.S_URL_PREFIX.length());
    }
965347
public static String getRequestLink(String url) {

        if (CmsStringUtil.isEmpty(url)) {
            return null;
        }
        int pos = url.indexOf(URL_DELIMITER);
        if (pos >= 0) {
            return url.substring(0, pos);
        }
        return url;

    }
1018448
@Override
    public List<ExtractResult> extract(String url, String html){
        List<ExtractResult> extractResults = new ArrayList<>();
        //同一个URL可能会有多个页面模板
        List<HtmlTemplate> htmlTemplates = extractRegular.getHtmlTemplate(url);
        if (htmlTemplates.isEmpty()) {
            LOGGER.debug("没有为此URL指定模板："+url);
            return extractResults;
        }
        try {
            Document doc = Jsoup.parse(html);
            Elements metas = doc.select("meta");
            String keywords = "";
            String description = "";
            for (Element meta : metas) {
                String name = meta.attr("name");
                if ("keywords".equals(name)) {
                    keywords = meta.attr("content");
                }
                if ("description".equals(name)) {
                    description = meta.attr("content");
                }
            }
            Set<String> tableNames = new HashSet<>();
            for (HtmlTemplate htmlTemplate : htmlTemplates) {
                if (tableNames.contains(htmlTemplate.getTableName())) {
                    LOGGER.debug("多个模板定义的tableName重复，这有可能会导致数据丢失，检查UrlPattern下定义的模板：" + htmlTemplate.getUrlPattern().getUrlPattern());
                    LOGGER.debug(htmlTemplates.toString());
                }
                tableNames.add(htmlTemplate.getTableName());
                try {
                    //按页面模板的定义对网页进行抽取
                    ExtractResult extractResult = extractHtmlTemplate(url, htmlTemplate, doc);
                    //同样的URL模式，因为改版等原因，可能会对应多套模板，多套模板中我们一般只需要抽取一套
                    if(!extractResult.getExtractFailLogs().isEmpty() || !extractResult.getExtractResultItems().isEmpty()) {
                        extractResult.setContent(html.getBytes("utf-8"));
                        extractResult.setEncoding("utf-8");
                        extractResult.setKeywords(keywords);
                        extractResult.setDescription(description);
                        extractResults.add(extractResult);
                    }else{
                        LOGGER.debug(url + " 的模板 " + htmlTemplate.getTemplateName() + " 未抽取到");
                    }
                } catch (Exception e) {
                    LOGGER.error("页面模板抽取失败：" + htmlTemplate.getTemplateName(), e);
                }
            }
        } catch (Exception e) {
            LOGGER.error("抽取网页出错: " + url, e);
        }
        return extractResults;
    }
1119927
protected URL extractURL(Document doc) throws IOException {
    String url = doc.getFirstChild().getTextContent();
    return (null == url || "".equals(url)) ? null : new URL(url);
  }