Downloads and parses all the webpages

    For Backend
355726
def _getCurrentj9Dict():
    
    urls = j9urlGenerator()
    j9Dict = {}
    for url in urls:
        d = _getDict(urllib.request.urlopen(url))
        if len(d) == 0:
            raise RuntimeError("Parsing failed, this is could require an update of the parser.")
        j9Dict.update(d)
    return j9Dict
1053623
def _fetch_all(cls, api_key, endpoint=None, offset=0, limit=25, **kwargs):
        
        output = []
        qp = kwargs.copy()
        limit = max(1, min(100, limit))
        maximum = kwargs.get('maximum')
        qp['limit'] = min(limit, maximum) if maximum is not None else limit
        qp['offset'] = offset
        more, total = None, None

        while True:
            entities, options = cls._fetch_page(
                api_key=api_key, endpoint=endpoint, **qp
            )
            output += entities
            more = options.get('more')
            limit = options.get('limit')
            offset = options.get('offset')
            total = options.get('total')

            if more is None:
                if total is None or offset is None:
                    break
                more = (limit + offset) < total

            if not more or (maximum is not None and len(output) >= maximum):
                break

            qp['limit'] = limit
            qp['offset'] = offset + limit

        return output
1110334
function fetch(url, options, callback) {
        if (typeof options === 'function') {
          callback = options;
          options = {};
        }
        options = options || {};
        options.callback = options.callback || 'callback';
        var target = document.head || document.getElementsByTagName('head')[0];
        var script = document.createElement('script');
        var jsonpCallback = 'jsonp_' + Embedo.utils.uuid();
        url += (~url.indexOf('?') ? '&' : '?') + options.callback + '=' + encodeURIComponent(jsonpCallback);
        url = url.replace('?&', '?');

        window[jsonpCallback] = function (data) {
          clear(jsonpCallback, script);
          callback(null, data);
        };

        script.type = 'text/javascript';
        script.defer = true;
        script.charset = 'UTF-8';
        script.onerror = function (err) {
          clear(jsonpCallback, script);
          return callback(err);
        };
        target.appendChild(script);
        script.src = url;

        function clear(jsonpCallback, script) {
          try {
            delete window[jsonpCallback];
          } catch (e) {
            window[jsonpCallback] = undefined;
          }
          if (script) {
            target.removeChild(script);
            script = undefined;
          }
        }
      }
724097
def __downloadPage(factory, *args, **kwargs):
    """"""

    # The Twisted API is weird:
    # 1) web.client.downloadPage() doesn't give us the HTTP headers
    # 2) there is no method that simply accepts a URL and gives you back
    #    a HTTPDownloader object

    #TODO: convert getPage() usage to something similar, too

    downloader = factory(*args, **kwargs)
    if downloader.scheme == 'https':
        from twisted.internet import ssl
        contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(downloader.host, downloader.port,
                           downloader, contextFactory)
    else:
        reactor.connectTCP(downloader.host, downloader.port,
                           downloader)
    return downloader
890726
def download_wiki():
    """"""
    ambiguous = [i for i in l.UNITS.items() if len(i[1]) > 1]
    ambiguous += [i for i in l.DERIVED_ENT.items() if len(i[1]) > 1]
    pages = set([(j.name, j.uri) for i in ambiguous for j in i[1]])

    print
    objs = []
    for num, page in enumerate(pages):

        obj = {'url': page[1]}
        obj['_id'] = obj['url'].replace('https://en.wikipedia.org/wiki/', '')
        obj['clean'] = obj['_id'].replace('_', ' ')

        print '---> Downloading %s (%d of %d)' % \
              (obj['clean'], num + 1, len(pages))

        obj['text'] = wikipedia.page(obj['clean']).content
        obj['unit'] = page[0]
        objs.append(obj)

    path = os.path.join(l.TOPDIR, 'wiki.json')
    os.remove(path)
    json.dump(objs, open(path, 'w'), indent=4, sort_keys=True)

    print '\n---> All done.\n'
1151708
def all(self, res):
        ""
        B = get_backend()
        return B.get_objects(B.get_concrete(res))
248907
def _load_backend(self, backend_uri, context):
        
        parsed = parse.urlparse(backend_uri)
        options = dict(parse.parse_qsl(parsed.query))

        try:
            backend = self._entry_points[self.BACKENDS_ENTRY_POINT][parsed.scheme].load()
        except KeyError:
            raise BackendNotFoundError(
                "The requested backend `%s` could not be found in the "
                "registered entry points. Perhaps you forgot to install the "
                "corresponding backend package?" % parsed.scheme
            )

        password = (parse.unquote(parsed.password)
                    if parsed.password
                    else parsed.password)

        return backend(
            username=parsed.username, password=password,
            hostname=parsed.hostname, port=parsed.port,
            path=parsed.path, options=options, context=context,
        )
219896
def read_backend(self):
        '''
        '''
        session = self.session
        if session is not None:
            if self._field:
                return session.model(self._field.model).read_backend
            else:
                return session.model(self).read_backend
483519
def _fetch(self, urls):
        
        responses = self._request_bulk(urls)
        for response in responses:
            try:
                soup = BeautifulSoup(response.content, 'html.parser',
                                     from_encoding="iso-8859-1")
                text = soup.get_text()
            except Exception:
                text = response.text
            self.data.append(text) # Opportunistic findings
        return responses
1321447
def load_pa11y_results(stdout, spider, url):
    
    if not stdout:
        return []

    results = json.loads(stdout.decode('utf8'))

    ignore_rules = ignore_rules_for_url(spider, url)
    for rule in ignore_rules:
        results = [
            result for result in results
            if not ignore_rule_matches_result(rule, result)
        ]
    return results
1108388
def download(self, sub_url):
        """"""
        response = requests.get(sub_url, headers=self.headers).text
        soup = BS(response, 'lxml')
        downlink = self.base_url+soup.select('.download a')[0]['href']
        data = requests.get(downlink, headers=self.headers)
        z = zipfile.ZipFile(cStringIO.StringIO(data.content))
        srt_files = [f.filename for f in z.filelist
                     if f.filename.rsplit('.')[-1].lower() in ['srt', 'ass']]
        z.extract(srt_files[0], '/tmp/')
        return srt_files[0]
125369
def Main(url):
    '''
    
    '''
    # The object of Web-Scraping.
    web_scrape = WebScraping()
    # Execute Web-Scraping.
    document = web_scrape.scrape(url)
    # The object of automatic summarization with N-gram.
    auto_abstractor = NgramAutoAbstractor()
    # n-gram object
    auto_abstractor.n_gram = Ngram()
    # n of n-gram
    auto_abstractor.n = 3
    # Set tokenizer. This is japanese tokenizer with MeCab.
    auto_abstractor.tokenizable_doc = MeCabTokenizer()
    # Object of abstracting and filtering document.
    abstractable_doc = TopNRankAbstractor()
    # Execute summarization.
    result_dict = auto_abstractor.summarize(document, abstractable_doc)

    # Output 3 summarized sentences.
    limit = 3
    i = 1
    for sentence in result_dict["summarize_result"]:
        print(sentence)
        if i >= limit:
            break
        i += 1
580042
function parseResponse({ browser, history, document, response }) {
  const window      = document.defaultView;
  window._request   = response.request;
  window._response  = response;
  history.updateLocation(window, response._url);

  const done = window._eventQueue.waitForCompletion();
  response
    ._consume()
    .then(function(body) {

      const contentType = response.headers.get('Content-Type') || '';
      const html        = getHTMLFromResponseBody(body, contentType);
      response.body     = html;
      document.write(html || '<html></html>');
      document.close();

      browser.emit('loaded', document);
      if (response.status >= 400)
        throw new Error(`Server returned status code ${response.status} from ${response.url}`);
      if (!document.documentElement)
        throw new Error(`Could not parse document at ${response.url}`);

    })
    .then(function() {

      // Handle meta refresh.  Automatically reloads new location and counts
      // as a redirect.
      //
      // If you need to check the page before refresh takes place, use this:
      //   browser.wait({
      //     function: function() {
      //       return browser.query('meta[http-equiv="refresh"]');
      //     }
      //   });
      const refresh = getMetaRefresh(document);

      if (refresh) {
        const { refreshTimeout } = refresh;
        const { refreshURL }     = refresh;

        window._eventQueue.setTimeout(function() {
          // Allow completion function to run
          window._eventQueue.enqueue(function() {
            window._eventQueue.enqueue(function() {
              // Count a meta-refresh in the redirects count.
              history.replace(refreshURL);
              // This results in a new window getting loaded
              const newWindow = history.current.window;
              newWindow.addEventListener('load', function() {
                ++newWindow._request._redirectCount;
              });
            });
          });
        }, refreshTimeout * 1000);
      }

    })
    .catch(function(error) {
      browser.emit('error', error);
    })
    .then(done);
}
728890
def getdb(backend=None, **kwargs):
    ''''''
    if isinstance(backend, BackendDataServer):
        return backend
    backend = backend or settings.DEFAULT_BACKEND
    if not backend:
        return None
    scheme, address, params = parse_backend(backend)
    params.update(kwargs)
    if 'timeout' in params:
        params['timeout'] = int(params['timeout'])
    return _getdb(scheme, address, params)
630058
def get_backends():
    
    IGNORE_DIRS = ["core", "tools", "utils"]

    global backends
    if backends is None:
        backends = {}
        i3pystatus_dir = os.path.dirname(i3pystatus.__file__)
        subdirs = [
            x
            for x in next(os.walk(i3pystatus_dir))[1]
            if not x.startswith("_") and x not in IGNORE_DIRS
        ]
        for subdir in subdirs:
            dirs = next(os.walk(os.path.join(i3pystatus_dir, subdir)))[2]
            backends.update(
                {
                    x[:-3]: "i3pystatus.%s.%s" % (subdir, x[:-3])
                    for x in dirs
                    if not x.startswith("_") and x.endswith(".py")
                }
            )
    return backends
635133
function downloadFromUrl(page, ph, url, callback) {
    async.waterfall([
        function (callback) {
            page.getCookies(function (cookie) {
                callback(null, page, ph, cookie);
            });
        },
        function (page, ph, cookies, callback) {
            var cookieString = helpers.cookieToHeader(cookies);
            var headers = { Cookie: cookieString };
            request.get({
                url: url,
                headers: headers,
                encoding: null
            }, function (error, response, downloadedBytes) {
                callback(error, page, ph, downloadedBytes);
            });
        }
    ], callback);
}
1049553
def fetch_pages(url, referer = nil, depth = nil)
      begin
        url = URI(url) unless url.is_a?(URI)
        pages = []
        get(url, referer) do |response, code, location, redirect_to, response_time|
          pages << Page.new(location, :body => response.body.dup,
                                      :code => code,
                                      :headers => response.to_hash,
                                      :referer => referer,
                                      :depth => depth,
                                      :redirect_to => redirect_to,
                                      :response_time => response_time)
        end

        return pages
      rescue Exception => e
        if verbose?
          puts e.inspect
          puts e.backtrace
        end
        return [Page.new(url, :error => e)]
      end
    end