Download the page and create the soup
773789
def soup(self):
        """"""
        try:
            return self._soup
        except AttributeError:
            url = client.get_url("/presentations/%s" % self.index)
            content = self.client.fetch_no_cache(url).decode('utf-8')
            self._soup = bs4.BeautifulSoup(content, "html.parser")

            return self._soup
715362
def __doDownloadPage(self, *args, **kwargs):
        
        logger.debug("download page: %r, %r", args, kwargs)

        return self.__clientDefer(downloadPage(*args, **kwargs))
729304
def get_soup(page=''):
    
    content = requests.get('%s/%s' % (BASE_URL, page)).text
    return BeautifulSoup(content)
843883
def download(cls, url, filename=None):
        
        return utility.download(url, cls.directory(), filename)
1093324
def download_url(url):
    ''''''
    if sys.version_info.major < 3:
        from urllib2 import urlopen as url_open
        from urllib2 import URLError as url_error
    else:
        from urllib.request import urlopen as url_open
        from urllib.error import URLError as url_error
    try:
        resp = url_open(url)
        headers = resp.info()
    except url_error as e:
        print('Error downloading %s' % url)
        return None
    return resp.read()
951614
function scrapeSearchPage(pageUrl) {

    return co(function *() {
        //Load the HTML page
        const $ = cheerio.load(yield request(pageUrl));

        //For each card, scrape some data
        const cards = Array.from($("#cardResults li")).map(el => {
            const $card = $(el);

            //Just scrape the database URL and the image for now
            const url = Url.resolve(SCRAPE_URL, $card.find("a").attr("href"));
            return {
                url: url,
                image: Url.resolve(SCRAPE_URL, $card.find("img").attr("src")),
                id: cardIdFromUrl(url)
            };
        });

        //Work out how many pages in total there are
        const totalText = $('#cards-load-more>div>span').text();
        const numPages = parseInt(/\d of (\d+)/.exec(totalText)[1]);

        return {
            numPages: numPages,
            cards: cards
        };

    })();
}
811995
def download_url(url):
    ''''''
    import urllib2
    try:
        resp = urllib2.urlopen(url)
        headers = resp.info()
    except urllib2.URLError as e:
        print('Error downloading %s' % url)
        return None
    return resp.read()
111401
def fetch(self):
        
        soup = self.session.get_results_soup()
        self.courses = CoursesList(soup)
438282
def download(url, target_file, chunk_size=4096):
    
    r = requests.get(url, stream=True)

    with open(target_file, 'w+') as out:
        # And this is why I love Armin Ronacher:
        with click.progressbar(r.iter_content(chunk_size=chunk_size),
                               int(r.headers['Content-Length'])/chunk_size,
                               label='Downloading...') as chunks:
            for chunk in chunks:
                out.write(chunk)
957671
def generate_pages_by_file():
    """"""
    from veripress import app
    from veripress.model import storage
    from veripress.model.parsers import get_standard_format_name
    from veripress.helpers import traverse_directory

    deploy_dir = get_deploy_dir()

    def copy_file(src, dst):
        makedirs(os.path.dirname(dst), mode=0o755, exist_ok=True)
        shutil.copyfile(src, dst)

    with app.app_context(), app.test_client() as client:
        root_path = os.path.join(app.instance_path, 'pages')
        for path in traverse_directory(root_path):
            # e.g. 'a/b/c/index.md'
            rel_path = os.path.relpath(path, root_path)
            # e.g. ('a/b/c/index', '.md')
            filename, ext = os.path.splitext(rel_path)
            if get_standard_format_name(ext[1:]) is not None:
                # is source of custom page
                rel_url = filename.replace(os.path.sep, '/') + '.html'
                page = storage.get_page(rel_url, include_draft=False)
                if page is not None:
                    # it's not a draft, so generate the html page
                    makedirs(os.path.join(deploy_dir,
                                          os.path.dirname(rel_path)),
                             mode=0o755, exist_ok=True)
                    with open(os.path.join(deploy_dir, filename + '.html'),
                              'wb') as f:
                        f.write(client.get('/' + rel_url).data)
                if app.config['PAGE_SOURCE_ACCESSIBLE']:
                    copy_file(path, os.path.join(deploy_dir, rel_path))
            else:
                # is other direct files
                copy_file(path, os.path.join(deploy_dir, rel_path))
335822
def download_from_url(url, path):
    """"""
    def process_response(r):
        chunk_size = 16 * 1024
        total_size = int(r.headers.get('Content-length', 0))
        with open(path, "wb") as file:
            with tqdm(total=total_size, unit='B',
                      unit_scale=1, desc=path.split('/')[-1]) as t:
                for chunk in r.iter_content(chunk_size):
                    if chunk:
                        file.write(chunk)
                        t.update(len(chunk))

    if 'drive.google.com' not in url:
        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, stream=True)
        process_response(response)
        return

    print('downloading from Google Drive; may take a few minutes')
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token
        response = session.get(url, stream=True)

    process_response(response)
120773
def download_content(**args):
	
	args = validate_args(**args)

	if not args['directory']:
		args['directory'] = args['query'].replace(' ', '-')

	print("Downloading {0} {1} files on topic {2} from {3} and saving to directory: {4}"
		.format(args['limit'], args['file_type'], args['query'], args['website'], args['directory']))
		

	links = search(args['query'], args['engine'], args['website'], args['file_type'], args['limit'])

	if args['parallel']:
		download_parallel(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])
	else:
		download_series(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])
1325899
def download(self, uuid, url, forced_file_name=None,
                 progress_bar=True, chunk_size=256,
                 directory=None, overwrite=False):
        
        self.last_req_time = None
        url = self.get_full_url(url)
        self.log.debug("download url : " + url)
        # Building request
        request = urllib2.Request(url)
        # request.add_header('Content-Type', 'application/json; charset=UTF-8')
        request.add_header('Accept', 'application/json,*/*;charset=UTF-8')
        # request start
        starttime = datetime.datetime.now()
        # doRequest
        resultq = urllib2.urlopen(request)
        code = resultq.getcode()
        file_name = uuid
        self.log.debug("ret code : '" + str(code) + "'")
        if code == 200:
            content_lenth = resultq.info().getheader('Content-Length')
            if not content_lenth:
                msg = "No content lengh header found !"
                self.log.debug(msg)
                progress_bar = False
            else:
                file_size = int(content_lenth.strip())
            if forced_file_name:
                file_name = forced_file_name
            else:
                content_dispo = resultq.info().getheader('Content-disposition')
                if content_dispo:
                    content_dispo = content_dispo.strip()
                    file_name = extract_file_name(content_dispo)
            if directory:
                if os.path.isdir(directory):
                    file_name = directory + "/" + file_name
            if os.path.isfile(file_name):
                if not overwrite:
                    cpt = 1
                    while 1:
                        if not os.path.isfile(file_name + "." + str(cpt)):
                            file_name += "." + str(cpt)
                            break
                        cpt += 1
                else:
                    self.log.warn("'%s' already exists. It was overwriten.",
                                  file_name)
            stream = None
            pbar = None
            if progress_bar:
                widgets = [FileTransferSpeed(), ' <<<', Bar(), '>>> ',
                           Percentage(), ' ', ETA()]
                pbar = ProgressBar(widgets=widgets, maxval=file_size)
                stream = FileWithCallback(file_name, 'w', pbar.update,
                                          file_size, file_name)
                pbar.start()
            else:
                stream = file(file_name, 'w')
            while 1:
                chunk = resultq.read(chunk_size)
                if not chunk:
                    break
                stream.write(chunk)
            stream.flush()
            stream.close()
            if pbar:
                pbar.finish()
        # request end
        endtime = datetime.datetime.now()
        self.last_req_time = str(endtime - starttime)
        self.log.debug("download url : %(url)s : request time : %(time)s",
                       {"url": url,
                        "time": self.last_req_time})
        return (file_name, self.last_req_time)
52598
def save(self):
        ''''''
        with open(self.filename, 'w') as plist_file:
            plist_file.write(str(self.soup))
838736
def download_url(url, save_as, iter_size=_default_iter_size, enable_verbose=True):
    
    msg = Messenger()
    if enable_verbose:
        msg.on()
    else:
        msg.off()

    msg.show("Downloading %s from %s..." % (save_as, url))

    with open(save_as, "wb") as f:
        response = requests.get(url, stream=True)

        if not response.ok:
            print("http get error!")
            return

        start_time = time.clock()
        downloaded_size = 0
        for block in response.iter_content(iter_size):
            if not block:
                break
            f.write(block)
            elapse = datetime.timedelta(seconds=(time.clock() - start_time))
            downloaded_size += sys.getsizeof(block)
            msg.show("    Finished %s, elapse %s." % (
                string_SizeInBytes(downloaded_size), elapse
            ))

    msg.show("    Complete!")
326145
def download(url, path=None, headers=None, session=None, show_progress=True,
             resume=True, auto_retry=True, max_rst_retries=5,
             pass_through_opts=None, cainfo=None, user_agent=None, auth=None):
    """"""
    hm = Homura(url, path, headers, session, show_progress, resume,
                auto_retry, max_rst_retries, pass_through_opts, cainfo,
                user_agent, auth)
    hm.start()
499435
def download_page(url, data=None):
    ''''''
    conn = urllib2.urlopen(url, data)
    resp = conn.read()
    conn.close()
    return resp