Manages the chunked downloading of a file given an url
169337
def download_file_helper(url, input_path):
    
    r = requests.get(url, stream=True)
    if r.status_code != 200:
        cli_log.error("Failed to download file: %s" % r.json()["message"])
    local_full_path = get_download_dest(input_path, r.url)
    original_filename = os.path.split(local_full_path)[-1]
    with open(local_full_path, "wb") as f:
        click.echo("Downloading {}".format(original_filename), err=True)
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()
    pprint("Successfully downloaded %s to %s" % (original_filename, local_full_path), True)
50957
def download(self,
                 url,
                 dest_path=None):
        
        if os.path.exists(dest_path):
            os.remove(dest_path)

        resp = get(url, stream=True)
        size = int(resp.headers.get("content-length"))
        label = "Downloading {filename} ({size:.2f}MB)".format(
            filename=os.path.basename(dest_path),
            size=size / float(self.chunk_size) / self.chunk_size
        )

        with open_file(dest_path, 'wb') as file:
            content_iter = resp.iter_content(chunk_size=self.chunk_size)
            with progressbar(content_iter,
                             length=size / self.chunk_size,
                             label=label) as bar:
                for chunk in bar:
                    if chunk:
                        file.write(chunk)
                        file.flush()
804535
def download(url, dest):
    """"""
    path = os.path.join(dest, url.split('/')[-1])
    r = requests.get(url, stream=True)
    r.raise_for_status()
    with open(path, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
    return path
197872
def download_file(url, dst_path):
    """"""
    request = requests.get(url, stream=True)
    with open(dst_path, 'wb') as downloaded_file:
        request.raw.decode_content = True
        shutil.copyfileobj(request.raw, downloaded_file)
1235549
def download(self, url, path=None, force=False):
        """"""
        logger.debug(str((url, path, force)))
        method = self._download
        return self._create_worker(method, url, path=path, force=force)
1098287
function httpDownload(targetPath, path, file)  {
    switch (configuredDownloadClient) {
        case "curl": return sh`curl -A curl --progress -o "${targetPath}/${file}" "https://transfer.sh/${path}/${file}"`;
        case "wget": return sh`wget --progress=dot -O "${targetPath}/${file}" "https://transfer.sh/${path}/${file}"`;
        case "fetch": return sh`fetch -q -o "${targetPath}/${file}" "https://transfer.sh/${path}/${file}"`;
        default: throw new Error("No configured download client");
    }
}
252255
def download_file(url, path=None, clobber=False):
    
    if path is None:
        local_filename = os.path.join(directory, url.split('/')[-1])
    else:
        local_filename = path

    if os.path.exists(local_filename) and not clobber:
        logging.info('{} exists; not downloading.'.format(local_filename))
        return local_filename
        
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)
                #f.flush() commented by recommendation from J.F.Sebastian
    return local_filename
864958
def download_chunk(self, url, headers):
        
        response = requests.get(url, headers=headers, stream=True)
        if response.status_code == SWIFT_EXPIRED_STATUS_CODE \
                or response.status_code == S3_EXPIRED_STATUS_CODE:
            raise DownloadInconsistentError(response.text)
        response.raise_for_status()
        self.actual_bytes_read = 0
        self._write_response_to_file(response)
        self._verify_download_complete()
304745
def hashed_download(url, temp, digest):
    """"""
    # Based on pip 1.4.1's URLOpener but with cert verification removed
    def opener():
        opener = build_opener(HTTPSHandler())
        # Strip out HTTPHandler to prevent MITM spoof:
        for handler in opener.handlers:
            if isinstance(handler, HTTPHandler):
                opener.handlers.remove(handler)
        return opener

    def read_chunks(response, chunk_size):
        while True:
            chunk = response.read(chunk_size)
            if not chunk:
                break
            yield chunk

    response = opener().open(url)
    path = join(temp, urlparse(url).path.split('/')[-1])
    actual_hash = sha256()
    with open(path, 'wb') as file:
        for chunk in read_chunks(response, 4096):
            file.write(chunk)
            actual_hash.update(chunk)

    actual_digest = actual_hash.hexdigest()
    if actual_digest != digest:
        raise HashError(url, path, actual_digest, digest)
    return path
789411
def download(self, url, path):
        """"""
        # original_url = url
#        print(url)
        qurl = QUrl(url)
        url = to_text_string(qurl.toEncoded(), encoding='utf-8')

        logger.debug(str((url, path)))
        if url in self._workers:
            while not self._workers[url].finished:
                return self._workers[url]

        worker = DownloadWorker(url, path)

        # Check download folder exists
        folder = os.path.dirname(os.path.abspath(path))
        if not os.path.isdir(folder):
            os.makedirs(folder)

        request = QNetworkRequest(qurl)
        self._head_requests[url] = request
        self._paths[url] = path
        self._workers[url] = worker
        self._manager.head(request)
        self._timer.start()

        return worker
363158
def download_file(url, file_name):
    

    logger.info("Downloading URL: %s", url)
    file_size = 0

    if not os.path.isfile(file_name):
        response = requests.get(url, stream=True)

        with open(file_name, "wb") as fp:
            if not response.ok:
                raise Exception("Download exception. Will fail.")

            for block in response.iter_content(1024):
                if not block:
                    break

                fp.write(block)
                file_size += len(block)

        logger.info("Download finished, size is %d bytes.", file_size)

    return file_size
271685
def file_download(user_session, request):
    
    logger.debug('Download file: %s', vars(request))
    response = user_session.session.get(
        request.href,
        params=request.params,
        headers=request.headers,
        stream=True)

    if response.status_code == 200:
        logger.debug('Streaming to file... Content length: %s', len(response.content))
        try:
            path = os.path.abspath(request.filename)
            logger.debug('Operation: %s, saving to file: %s', request.href, path)

            with open(path, "wb") as handle:
                for chunk in response.iter_content(chunk_size=1024):
                    if chunk:
                        handle.write(chunk)
                        handle.flush()
        except IOError as e:
            raise IOError('Error attempting to save to file: {}'.format(e))

        result = SMCResult(response, user_session=user_session)
        result.content = path
        return result
    else:
        raise SMCOperationFailure(response)
317817
function download (data, filename) {
  const element = getElement();
  const url = getObjectUrl(data);

  element.setAttribute('href', url);
  element.setAttribute('download', filename);
  document.body.appendChild(element);
  element.click();
  document.body.removeChild(element);

  setTimeout(function () {
    URL.revokeObjectURL(url);
  }, 100);
}
97714
def _chunk_iter(self):
        """"""
        for chunk_offset in self._chunk_offsets():
            yield self._download_chunk(chunk_offset=chunk_offset,
                                       chunk_size=self._chunk_size)
713694
def _download_without_backoff(url, as_file=True, method='GET', **kwargs):
    
    # Make requests consistently hashable for caching.
    # 'headers' is handled by requests itself.
    # 'cookies' and 'proxies' contributes to headers.
    # 'files' and 'json' contribute to data.
    for k in ['data', 'params']:
        if k in kwargs and isinstance(kwargs[k], dict):
            kwargs[k] = OrderedDict(sorted(kwargs[k].items()))

    kwargs_copy = dict(kwargs)
    if not _is_url_in_cache(method, url, **kwargs):
        now = datetime.datetime.now()
        _rate_limit_for_url(url, now)
        _rate_limit_touch_url(url, now)

    L.info("Download {}".format(url))
    if 'timeout' not in kwargs_copy:
        kwargs_copy['timeout'] = _TIMEOUT
    if 'headers' in kwargs_copy:
        head_dict = CaseInsensitiveDict(kwargs_copy['headers'])
        if 'user-agent' not in head_dict:
            head_dict['user-agent'] = _USER_AGENT
        kwargs_copy['headers'] = head_dict
    else:
        kwargs_copy['headers'] = CaseInsensitiveDict({'user-agent': _USER_AGENT})

    response = requests.request(method, url, **kwargs_copy)

    if logging.getLogger().isEnabledFor(logging.DEBUG):
        # This can be slow on large responses, due to chardet.
        L.debug('"{}"'.format(response.text))

    response.raise_for_status()

    if as_file:
        return BytesIO(response.content)
    else:
        return response
335822
def download_from_url(url, path):
    """"""
    def process_response(r):
        chunk_size = 16 * 1024
        total_size = int(r.headers.get('Content-length', 0))
        with open(path, "wb") as file:
            with tqdm(total=total_size, unit='B',
                      unit_scale=1, desc=path.split('/')[-1]) as t:
                for chunk in r.iter_content(chunk_size):
                    if chunk:
                        file.write(chunk)
                        t.update(len(chunk))

    if 'drive.google.com' not in url:
        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, stream=True)
        process_response(response)
        return

    print('downloading from Google Drive; may take a few minutes')
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token
        response = session.get(url, stream=True)

    process_response(response)
536526
function wget(fileURL, callback) {
		var
			chunks = [],
			isHTTPS = fileURL.indexOf('https') === 0,
			client = isHTTPS ? https : http,
			options = fileURL
		;
		if (isHTTPS) {
			options = url.parse(fileURL);
			options.rejectUnauthorized = false;
			options.agent = new https.Agent(options);
		}
		client.get(options, function (res) {
			res.on('data', function (chunk) {
				chunks.push(chunk);
			}).on('end', function () {
				callback(null, joinBuffers(chunks));
			});
		});
	}