Scrape webpage content.
18127
def scrape(args):
    """"""
    try:
        base_dir = os.getcwd()
        if args['out'] is None:
            args['out'] = []

        # Detect whether to save to a single or multiple files
        detect_output_type(args)

        # Split query input into local files and URLs
        split_input(args)

        if args['urls']:
            # Add URL extensions and schemes and update query and URLs
            urls_with_exts = [utils.add_url_suffix(x) for x in args['urls']]
            args['query'] = [utils.add_protocol(x) if x in args['urls'] else x
                             for x in urls_with_exts]
            args['urls'] = [x for x in args['query'] if x not in args['files']]

        # Print error if attempting to convert local files to HTML
        if args['files'] and args['html']:
            sys.stderr.write('Cannot convert local files to HTML.\n')
            args['files'] = []

        # Instantiate web crawler if necessary
        crawler = None
        if args['crawl'] or args['crawl_all']:
            crawler = Crawler(args)

        if args['single']:
            return write_single_file(args, base_dir, crawler)
        elif args['multiple']:
            return write_multiple_files(args, base_dir, crawler)

    except (KeyboardInterrupt, Exception):
        if args['html']:
            try:
                os.chdir(base_dir)
            except OSError:
                pass
        else:
            utils.remove_part_files()
        raise
109982
def links(self):
        """"""
        for match in self._href_re.finditer(self.content):
            url = match.group(1) or match.group(2) or match.group(3)
            url = self.clean_link(urlparse.urljoin(self.base_url, url))
            yield Link(url, self)
824757
def soupify(self):
        
        soup = BeautifulSoup(self.content_page.content, 'lxml')
        if self.page == 'torrent_project':
            main = soup.find('div', {'id': 'similarfiles'})
            if self.mode_search == 'best_rated':
                rated_url = self.domain + \
                    main.find(href=re.compile('torrent.html'))['href']
                self.get_magnet(rated_url)
            else:
                divs = main.find_all('div', limit=30)[2:]
                self.elements = list(
                    zip(*[d.find_all('span', recursive=False)
                          for d in divs]))  # Torrents

        elif self.page == 'the_pirate_bay':
            main = soup.find('table', {'id': 'searchResult'})
            if self.mode_search == 'best_rated':
                rated_url = self.domain + \
                    main.find('a', href=re.compile('torrent'))['href']
                self.get_magnet(rated_url)
            else:
                trs = main.find_all('tr', limit=30)[1:]
                self.elements = list(
                    zip(*[tr.find_all('td', recursive=False)[1:]
                          for tr in trs]))  # Magnets

        elif self.page == '1337x':
            main = soup.find('table', {'class': 'table'})
            if self.mode_search == 'best_rated':
                rated_url = self.domain + \
                    main.find('a', href=re.compile('torrent'))['href']
                self.get_magnet(rated_url)
            else:
                trs = main.find_all('tr', limit=30)[1:]
                self.elements = list(
                    zip(*([tr.find_all('td', recursive=False)[:-1]
                           for tr in trs])))  # Torrents

        elif self.page == 'eztv':
            main = soup.find_all('table', {'class': 'forum_header_border'})[2]
            if self.mode_search == 'best_rated':
                self.magnet = main.find('a', href=re.compile('magnet'))['href']
            else:
                trs = main.find_all('tr', limit=30)[2:]
                self.elements = list(
                    zip(*([tr.find_all('td', recursive=False)[1:-1]
                           for tr in trs])))  # Magnets

        elif self.page == 'limetorrents':
            main = soup.find('table', {'class': 'table2'})
            if self.mode_search == 'best_rated':
                self.magnet = main.find(
                    'a', href=re.compile('torrent'))['href']
            else:
                trs = main.find_all('tr', limit=30)[1:]
                self.elements = list(
                    zip(*([tr.find_all('td', recursive=False)[:-1]
                           for tr in trs])))  # Magnets

        elif self.page == 'isohunt':
            main = soup.find('table', {'class': 'table'})
            if self.mode_search == 'best_rated':
                rated_url = self.domain + \
                    main.find('a', href=re.compile(
                        'torrent_details'))['href']
                self.get_magnet(rated_url)
            else:
                trs = main.find_all('tr', limit=30)[1:-1]
                self.elements = list(
                    zip(*([tr.find_all('td', recursive=False)[1:-1]
                           for tr in trs])))  # Torrent
        else:
            print('Cannot soupify current page. Try again.')
1324147
def getPageContent(url, session, max_content_bytes=MaxContentBytes):
    """"""
    check_robotstxt(url, session)
    # read page data
    try:
        page = urlopen(url, session, max_content_bytes=max_content_bytes)
    except IOError:
        page = urlopen(url, session, max_content_bytes=max_content_bytes)
    data = page.text
    tries = MaxRetries
    while not isValidPageContent(data) and tries > 0:
        time.sleep(RetryPauseSeconds)
        page = urlopen(url, session, max_content_bytes=max_content_bytes)
        data = page.text
        tries -= 1
    if not isValidPageContent(data):
        raise ValueError("Got invalid page content from %s: %r" % (url, data))
    out.debug(u"Got page content %r" % data, level=3)
    return data
413256
def scrape(self, url):
        '''
        
        '''
        if isinstance(url, str) is False:
            raise TypeError("The type of url must be str.")

        if self.readable_web_pdf is not None and self.readable_web_pdf.is_pdf_url(url) is True:
            web_data = self.readable_web_pdf.url_to_text(url)
        else:
            web_data = ""
            req = urllib.request.Request(url=url)
            with urllib.request.urlopen(req) as f:
                web = f.read().decode('utf-8')
                dom = pq(web)
                [dom(remove_object).remove() for remove_object in self.__remove_object_list]

                for dom_object in self.__dom_object_list:
                    web_data += dom(dom_object).text()

        sleep(1)
        return web_data
809437
def scrape_file(file_string, line)
      lines = file_string.split("\n")
      saved = []
      i = line -2
      while lines[i] =~ /^\s*#\s*(\S+)/ && i >= 0
        saved << lines[i]
        i -= 1
      end

      saved.empty? ? {} : splitter(saved.reverse)
    end
383793
def explicit_rel_links(self, rels=('homepage', 'download')):
        """"""
        for match in self._rel_re.finditer(self.content):
            found_rels = match.group(1).lower().split()
            for rel in rels:
                if rel in found_rels:
                    break
            else:
                continue
            match = self._href_re.search(match.group(0))
            if not match:
                continue
            url = match.group(1) or match.group(2) or match.group(3)
            url = self.clean_link(urlparse.urljoin(self.base_url, url))
            yield Link(url, self)
47559
def get_content (self):
        """"""
        if self.data is None:
            log.debug(LOG_CHECK, "Get content of %r", self.url)
            t = time.time()
            self.data = self.read_content()
            self.size = len(self.data)
            self.dltime = time.time() - t
            if self.size == 0:
                self.add_warning(_("Content size is zero."),
                             tag=WARN_URL_CONTENT_SIZE_ZERO)
            else:
                self.aggregate.add_downloaded_bytes(self.size)
        return self.data
782796
def Main(url):
    '''
    
    '''
    # The object of Web-Scraping.
    web_scrape = WebScraping()
    # Execute Web-Scraping.
    document = web_scrape.scrape(url)
    # The object of NLP.
    nlp_base = NlpBase()
    # Set tokenizer. This is japanese tokenizer with MeCab.
    nlp_base.tokenizable_doc = MeCabTokenizer()

    sentence_list = nlp_base.listup_sentence(document)

    batch_size = 10
    if len(sentence_list) < batch_size:
        raise ValueError("The number of extracted sentences is insufficient.")

    all_token_list = []
    for i in range(len(sentence_list)):
        nlp_base.tokenize(sentence_list[i])
        all_token_list.extend(nlp_base.token)
        sentence_list[i] = nlp_base.token
    
    vectorlizable_sentence = LSTMRTRBM()
    vectorlizable_sentence.learn(
        sentence_list=sentence_list, 
        token_master_list=list(set(all_token_list)),
        hidden_neuron_count=1000,
        batch_size=batch_size,
        learning_rate=1e-03,
        seq_len=5
    )
    test_list = sentence_list[:batch_size]
    feature_points_arr = vectorlizable_sentence.vectorize(test_list)

    print("Feature points (Top 5 sentences):")
    print(feature_points_arr)
951614
function scrapeSearchPage(pageUrl) {

    return co(function *() {
        //Load the HTML page
        const $ = cheerio.load(yield request(pageUrl));

        //For each card, scrape some data
        const cards = Array.from($("#cardResults li")).map(el => {
            const $card = $(el);

            //Just scrape the database URL and the image for now
            const url = Url.resolve(SCRAPE_URL, $card.find("a").attr("href"));
            return {
                url: url,
                image: Url.resolve(SCRAPE_URL, $card.find("img").attr("src")),
                id: cardIdFromUrl(url)
            };
        });

        //Work out how many pages in total there are
        const totalText = $('#cards-load-more>div>span').text();
        const numPages = parseInt(/\d of (\d+)/.exec(totalText)[1]);

        return {
            numPages: numPages,
            cards: cards
        };

    })();
}
1089496
def _fetch(self):
        """"""
        url = client.get_url("/presentations/" + self.id)
        content = self.client.fetch_no_cache(url).decode('utf-8')
        return bs4.BeautifulSoup(content, "html.parser")
33569
def _get_html_contents(html):
    """"""
    parser = MyHTMLParser()
    parser.feed(html)
    if parser.is_code:
        return ('code', parser.data.strip())
    elif parser.is_math:
        return ('math', parser.data.strip())
    else:
        return '', ''
729304
def get_soup(page=''):
    
    content = requests.get('%s/%s' % (BASE_URL, page)).text
    return BeautifulSoup(content)
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
1110606
def _page_html(page):
    
    out = []
    out.append(_html_title(page))
    out.append(_html_image(page))
    out.append(page.data.get('extract'))
    return "\n".join([x for x in out if x])
305904
def fetch(self):
        """"""
        self.content = self.client.get_content(
            uri=self.uri
        )
        self.hash = hashlib.sha256(
            self.content
        ).hexdigest()
886213
def make_scrape_request(session, url, mode='get', data=None):
    """"""
    try:
        html = session.request(mode, url, data=data)
    except RequestException:
        raise VooblyError('failed to connect')
    if SCRAPE_FETCH_ERROR in html.text:
        raise VooblyError('not logged in')
    if html.status_code != 200 or SCRAPE_PAGE_NOT_FOUND in html.text:
        raise VooblyError('page not found')
    return bs4.BeautifulSoup(html.text, features='lxml')