Readability Article Extraction from URL
165437
public function readability( $url )
	{
		$data = $this->get_call('/readability', ['url' => $url ] );

		return ( !empty($data['readability']) ) ? $data['readability'] : null;
	}
584479
def _normalize_answer(text: str) -> str:
    """"""

    parts = [_white_space_fix(_remove_articles(_normalize_number(_remove_punc(_lower(token)))))
             for token in _tokenize(text)]
    parts = [part for part in parts if part.strip()]
    normalized = ' '.join(parts).strip()
    return normalized
496370
def extracturls(mesg):
    
    lines = NLRE.split(mesg)

    # The number of lines of context above to provide.
    # above_context = 1
    # The number of lines of context below to provide.
    # below_context = 1

    # Plan here is to first transform lines into the form
    # [line_fragments] where each fragment is a chunk as
    # seen by parse_text_urls.  Note that this means that
    # lines with more than one entry or one entry that's
    # a URL are the only lines containing URLs.

    linechunks = [parse_text_urls(l) for l in lines]

    return extract_with_context(linechunks,
                                lambda chunk: len(chunk) > 1 or
                                (len(chunk) == 1 and chunk[0].url is not None),
                                1, 1)
816805
def fetchText(cls, url, data, textSearch, optional):
        """"""
        if textSearch:
            match = textSearch.search(data[0])
            if match:
                text = match.group(1)
                out.debug(u'matched text %r with pattern %s' % (text, textSearch.pattern))
                return unescape(text).strip()
            if optional:
                return None
            else:
                raise ValueError("Pattern %s not found at URL %s." % (textSearch.pattern, url))
        else:
            return None
1147849
def guess_url(url):
    
    if url.lower().startswith("www."):
        # syntactic sugar
        return "http://%s" % url
    elif url.lower().startswith("ftp."):
        # syntactic sugar
        return "ftp://%s" % url
    return url
1211658
def extractor(url):
    """"""
    response = requester(url, main_url, delay, cook, headers, timeout, host, proxies, user_agents, failed, processed)
    if clone:
        mirror(url, response)
    matches = rhref.findall(response)
    for link in matches:
        # Remove everything after a "#" to deal with in-page anchors
        link = link[1].replace('\'', '').replace('"', '').split('#')[0]
        # Checks if the URLs should be crawled
        if is_link(link, processed, files):
            if link[:4] == 'http':
                if link.startswith(main_url):
                    verb('Internal page', link)
                    internal.add(link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:2] == '//':
                if link.split('/')[2].startswith(host):
                    verb('Internal page', link)
                    internal.add(schema + '://' + link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:1] == '/':
                verb('Internal page', link)
                internal.add(remove_file(url) + link)
            else:
                verb('Internal page', link)
                usable_url = remove_file(url)
                if usable_url.endswith('/'):
                    internal.add(usable_url + link)
                elif link.startswith('/'):
                    internal.add(usable_url + link)
                else:
                    internal.add(usable_url + '/' + link)

    if not only_urls:
        intel_extractor(url, response)
        js_extractor(response)
    if args.regex and not supress_regex:
        regxy(args.regex, response, supress_regex, custom)
    if api:
        matches = rentropy.findall(response)
        for match in matches:
            if entropy(match) >= 4:
                verb('Key', match)
                keys.add(url + ': ' + match)
803527
def _extract_from_url(self, url):
        """"""

        # Regex by Newspaper3k  - https://github.com/codelucas/newspaper/blob/master/newspaper/urls.py
        m = re.search(re_pub_date, url)
        if m:
            return self.parse_date_str(m.group(0))
        return None
20520
def article_parse(self, response, rss_title=None):
        
        if not self.helper.parse_crawler.content_type(response):
            return

        yield self.helper.parse_crawler.pass_to_pipeline_if_article(
            response, self.ignored_allowed_domain, self.original_url,
            rss_title)
846286
def text_extract(path, password=None):
    """"""
    pdf = Info(path, password).pdf

    return [pdf.getPage(i).extractText() for i in range(pdf.getNumPages())]
858768
def extract_paragraphs(xml_string):
    """"""
    assert isinstance(xml_string, str)
    xml_tree = ET.XML(xml_string.encode('utf-8'), parser=UTB())
    full_text = xml_tree.find('article:originalText', elsevier_ns)
    if full_text is None:
        logger.info('Could not find full text element article:originalText')
        return None
    article_body = _get_article_body(full_text)
    if article_body:
        return article_body
    raw_text = _get_raw_text(full_text)
    if raw_text:
        return [raw_text]
    return None
36680
def url_decode(url):
        
        return re.compile('%([0-9a-fA-F]{2})', re.M).sub(lambda m: chr(int(m.group(1), 16)), url)
485415
def parse_link(value):
    

    links = []

    replace_chars = ' \'"'

    value = value.strip(replace_chars)
    if not value:
        return links

    for val in re.split(', *<', value):
        try:
            url, params = val.split(';', 1)
        except ValueError:
            url, params = val, ''

        link = {'url': url.strip('<> \'"')}

        for param in params.split(';'):
            try:
                key, value = param.split('=')
            except ValueError:
                break

            link[key.strip(replace_chars)] = value.strip(replace_chars)

        links.append(link)

    return links
737580
private ResourceReference parseDocumentLink(StringBuilder content)
    {
        // Extract any query string.
        String queryString = null;
        String text = parseElementAfterString(content, SEPARATOR_QUERYSTRING);
        if (text != null) {
            queryString = removeEscapesFromExtraParts(text);
        }

        // Extract any anchor.
        String anchor = null;
        text = parseElementAfterString(content, SEPARATOR_ANCHOR);
        if (text != null) {
            anchor = removeEscapesFromExtraParts(text);
        }

        // Make sure to unescape the remaining reference string.
        String unescapedReferenceString = removeEscapesFromReferencePart(content.toString());
        // Parse the string as an untyped link reference.
        ResourceReference reference = untypedLinkReferenceParser.parse(unescapedReferenceString);
        reference.setTyped(false);

        // Set any previously extracted parameters.
        if (StringUtils.isNotBlank(queryString)) {
            reference.setParameter(DocumentResourceReference.QUERY_STRING, queryString);
        }
        if (StringUtils.isNotBlank(anchor)) {
            reference.setParameter(DocumentResourceReference.ANCHOR, anchor);
        }

        return reference;
    }
782796
def Main(url):
    '''
    
    '''
    # The object of Web-Scraping.
    web_scrape = WebScraping()
    # Execute Web-Scraping.
    document = web_scrape.scrape(url)
    # The object of NLP.
    nlp_base = NlpBase()
    # Set tokenizer. This is japanese tokenizer with MeCab.
    nlp_base.tokenizable_doc = MeCabTokenizer()

    sentence_list = nlp_base.listup_sentence(document)

    batch_size = 10
    if len(sentence_list) < batch_size:
        raise ValueError("The number of extracted sentences is insufficient.")

    all_token_list = []
    for i in range(len(sentence_list)):
        nlp_base.tokenize(sentence_list[i])
        all_token_list.extend(nlp_base.token)
        sentence_list[i] = nlp_base.token
    
    vectorlizable_sentence = LSTMRTRBM()
    vectorlizable_sentence.learn(
        sentence_list=sentence_list, 
        token_master_list=list(set(all_token_list)),
        hidden_neuron_count=1000,
        batch_size=batch_size,
        learning_rate=1e-03,
        seq_len=5
    )
    test_list = sentence_list[:batch_size]
    feature_points_arr = vectorlizable_sentence.vectorize(test_list)

    print("Feature points (Top 5 sentences):")
    print(feature_points_arr)
891763
def extract_article
      api_response = request.get article_endpoint, token: Diffbot.token, url: @url
      @response = JSON.parse(api_response.body)
    end
479511
def _extract_subscription_url(url):
    
    match = re.match(r".*/subscriptions/[a-f0-9-]+/", url, re.IGNORECASE)
    if not match:
        raise ValueError("Unable to extract subscription ID from URL")
    return match.group(0)
1138322
def url_to_text(self, url):
        '''
        

        '''
        path, headers = urllib.request.urlretrieve(url)
        return self.path_to_text(path)