Tries to scrape the miniaturized version from vandale.nl.
503342
def miniaturize(self):
		'''  '''

		element = self._first('NN')
		if element:
			if re.search('verkleinwoord: (\w+)', element, re.U):
				return re.findall('verkleinwoord: (\w+)', element, re.U)
			else:
				return ['']
		return [None]
461412
function ScrapeError(msg, response, bodyString) {
  Error.call(this, msg);

  this.response = response;
  this.bodyString = bodyString;
}
548336
def scrape_document(self, item_session: ItemSession):
        ''''''
        self.event_dispatcher.notify(
            PluginFunctions.get_urls, item_session
        )

        if not self._document_scraper:
            return

        demux_info = self._document_scraper.scrape_info(
            item_session.request, item_session.response,
            item_session.url_record.link_type
        )

        num_inline_urls = 0
        num_linked_urls = 0

        for scraper, scrape_result in demux_info.items():
            new_inline, new_linked = self._process_scrape_info(
                scraper, scrape_result, item_session
            )
            num_inline_urls += new_inline
            num_linked_urls += new_linked

        _logger.debug('Candidate URLs: inline={0} linked={1}',
                      num_inline_urls, num_linked_urls
        )
1326226
def scrape_mixcloud_url(mc_url, num_tracks=sys.maxsize, folders=False, custom_path=''):
    

    try:
        data = get_mixcloud_data(mc_url)
    except Exception as e:
        puts_safe(colored.red("Problem downloading ") + mc_url)
        print(e)
        return []

    filenames = []

    track_artist = sanitize_filename(data['artist'])
    track_title = sanitize_filename(data['title'])
    track_filename = track_artist + ' - ' + track_title + data['mp3_url'][-4:]

    if folders:
        track_artist_path = join(custom_path, track_artist)
        if not exists(track_artist_path):
            mkdir(track_artist_path)
        track_filename = join(track_artist_path, track_filename)
        if exists(track_filename):
            puts_safe(colored.yellow("Skipping") + colored.white(': ' + data['title'] + " - it already exists!"))
            return []
    else:
        track_filename = join(custom_path, track_filename)

    puts_safe(colored.green("Downloading") + colored.white(
        ': ' + data['artist'] + " - " + data['title'] + " (" + track_filename[-4:] + ")"))
    download_file(data['mp3_url'], track_filename)
    if track_filename[-4:] == '.mp3':
        tag_file(track_filename,
                 artist=data['artist'],
                 title=data['title'],
                 year=data['year'],
                 genre="Mix",
                 artwork_url=data['artwork_url'])
    filenames.append(track_filename)

    return filenames
590654
def scrape(self, request, response, link_type=None):
        ''''''
        for scraper in self._document_scrapers:
            scrape_result = scraper.scrape(request, response, link_type)

            if scrape_result is None:
                continue

            if scrape_result.link_contexts:
                return scrape_result
1211658
def extractor(url):
    """"""
    response = requester(url, main_url, delay, cook, headers, timeout, host, proxies, user_agents, failed, processed)
    if clone:
        mirror(url, response)
    matches = rhref.findall(response)
    for link in matches:
        # Remove everything after a "#" to deal with in-page anchors
        link = link[1].replace('\'', '').replace('"', '').split('#')[0]
        # Checks if the URLs should be crawled
        if is_link(link, processed, files):
            if link[:4] == 'http':
                if link.startswith(main_url):
                    verb('Internal page', link)
                    internal.add(link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:2] == '//':
                if link.split('/')[2].startswith(host):
                    verb('Internal page', link)
                    internal.add(schema + '://' + link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:1] == '/':
                verb('Internal page', link)
                internal.add(remove_file(url) + link)
            else:
                verb('Internal page', link)
                usable_url = remove_file(url)
                if usable_url.endswith('/'):
                    internal.add(usable_url + link)
                elif link.startswith('/'):
                    internal.add(usable_url + link)
                else:
                    internal.add(usable_url + '/' + link)

    if not only_urls:
        intel_extractor(url, response)
        js_extractor(response)
    if args.regex and not supress_regex:
        regxy(args.regex, response, supress_regex, custom)
    if api:
        matches = rentropy.findall(response)
        for match in matches:
            if entropy(match) >= 4:
                verb('Key', match)
                keys.add(url + ': ' + match)
1339739
def _build_document_scrapers(cls, session: AppSession):
        '''
        '''
        html_parser = session.factory['HTMLParser']
        element_walker = session.factory.new('ElementWalker')

        scrapers = [
            session.factory.new(
                'HTMLScraper',
                html_parser,
                element_walker,
                followed_tags=session.args.follow_tags,
                ignored_tags=session.args.ignore_tags,
                only_relative=session.args.relative,
                robots=session.args.robots,
                encoding_override=session.args.remote_encoding,
            ),
        ]

        if 'css' in session.args.link_extractors:
            css_scraper = session.factory.new(
                'CSSScraper',
                encoding_override=session.args.remote_encoding,
            )
            scrapers.append(css_scraper)
            element_walker.css_scraper = css_scraper

        if 'javascript' in session.args.link_extractors:
            javascript_scraper = session.factory.new(
                'JavaScriptScraper',
                encoding_override=session.args.remote_encoding,
            )
            scrapers.append(javascript_scraper)
            element_walker.javascript_scraper = javascript_scraper

        if session.args.sitemaps:
            scrapers.append(session.factory.new(
                'SitemapScraper', html_parser,
                encoding_override=session.args.remote_encoding,
            ))

        return scrapers
1118333
def _scrape_document(self):
        ''''''
        mock_response = self._new_mock_response(
            self._response, self._get_temp_path('phantom', '.html')
        )

        self._item_session.request = self._request
        self._item_session.response = mock_response

        self._processing_rule.scrape_document(item_session)

        if mock_response.body:
            mock_response.body.close()
335971
function scrapVideo(window){
   var $ = window.$;
   var url = window.location.href;
   var thumbs = [];

   // Open Graph protocol by Facebook: <meta property="og:video" content="(*)"/>
   $('meta').each(function(){
      var property = $(this).attr('property');
      var content = $(this).attr('content');
      if(property === 'og:video' && content){
         thumbs.push(utils.toURL(content));
      }
   });

   $('video, embed').each(function(){
      var src = $(this).attr('src');
      if(src) thumbs.push(utils.toURL(src,url));
   });

   return thumbs;
}
1303483
function scrapeMetadata (html, rules) {
  rules = rules || RULES
  const keys = Object.keys(rules)
  const $ = cheerio.load(html)
  const promises = keys.map(function (key) {
    return scrapeMetadatum($, '', rules[key])
  })

  return Promise.all(promises).then(function (values) {
    return keys.reduce(function (memo, key, i) {
      memo[key] = values[i]
      return memo
    }, {})
  })
}
186464
def extract(self, url=None, raw_html=None):
        
        cc = CrawlCandidate(self.config, url, raw_html)
        return self.crawl(cc)
236235
function getScraperResults (SCRAPER, urls, htmls) {
  return Promise.all(
    urls.map((url, i) => {
      const html = htmls[i]
      const name = SCRAPER.name
      const Module = require(name)
      return SCRAPER.scrape(Module, url, html).then(metadata => {
        return SCRAPER.normalize(metadata)
      })
    })
  )
}
530980
def scrape_from_list(list, blk)
      puts "Scraping #{list.size} instances" if @verbose
      list.each_with_index.map do |instance_url, instance_index|
        instance_resps = get_instance instance_url, nil, :instance_index => instance_index
        instance_resps.each_with_index.map do |instance_resp, pagination_index|
          blk.call(instance_resp, instance_url, instance_index, pagination_index)
        end
      end.flatten(1)
    end
886213
def make_scrape_request(session, url, mode='get', data=None):
    """"""
    try:
        html = session.request(mode, url, data=data)
    except RequestException:
        raise VooblyError('failed to connect')
    if SCRAPE_FETCH_ERROR in html.text:
        raise VooblyError('not logged in')
    if html.status_code != 200 or SCRAPE_PAGE_NOT_FOUND in html.text:
        raise VooblyError('page not found')
    return bs4.BeautifulSoup(html.text, features='lxml')
1047356
function collectResources(a, obj, tagName) {
        Array.prototype
            .forEach
            .call(a.ownerDocument.getElementsByTagName(tagName), function(r) {
                // Get canonical URL
                a.href = r.currentSrc || r.src || r.getAttribute("xlink:href") || r.href;

                // only get external resource
                if (a.href.match(/^https?:\/\//)) {
                    obj[a.href] = r;
                }
            });
    }
607048
def Main(url, similarity_mode="TfIdfCosine", similarity_limit=0.75):
    '''
    
    '''
    # The object of Web-scraping.
    web_scrape = WebScraping()
    # Set the object of reading PDF files.
    web_scrape.readable_web_pdf = WebPDFReading()
    # Execute Web-scraping.
    document = web_scrape.scrape(url)

    if similarity_mode == "TfIdfCosine":
        # The object of `Similarity Filter`. 
        # The similarity observed by this object is so-called cosine similarity of Tf-Idf vectors.
        similarity_filter = TfIdfCosine()

    elif similarity_mode == "Dice":
        # The object of `Similarity Filter`.
        # The similarity observed by this object is the Dice coefficient.
        similarity_filter = Dice()

    elif similarity_mode == "Jaccard":
        # The object of `Similarity Filter`.
        # The similarity observed by this object is the Jaccard coefficient.
        similarity_filter = Jaccard()
    
    elif similarity_mode == "Simpson":
        # The object of `Similarity Filter`.
        # The similarity observed by this object is the Simpson coefficient.
        similarity_filter = Simpson()
    
    else:
        raise ValueError()


    # The object of the NLP.
    nlp_base = NlpBase()
    # Set tokenizer. This is japanese tokenizer with MeCab.
    nlp_base.tokenizable_doc = MeCabTokenizer()
    # Set the object of NLP.
    similarity_filter.nlp_base = nlp_base
    # If the similarity exceeds this value, the sentence will be cut off.
    similarity_filter.similarity_limit = similarity_limit

    # The object of automatic sumamrization.
    auto_abstractor = AutoAbstractor()
    # Set tokenizer. This is japanese tokenizer with MeCab.
    auto_abstractor.tokenizable_doc = MeCabTokenizer()
    # Object of abstracting and filtering document.
    abstractable_doc = TopNRankAbstractor()
    # Execute summarization.
    result_dict = auto_abstractor.summarize(document, abstractable_doc, similarity_filter)
    # Output summarized sentence.
    [print(result_dict["summarize_result"][i]) for i in range(len(result_dict["summarize_result"])) if i < 3]
276842
def scrape(&blk)
      self.url_array = self.get_index unless self.url_array
      blk = Proc.new{|x| x} if blk.nil?
      self.scrape_from_list(self.url_array, blk)
    end