Download a file from an HTTP source.

    :param source: URL to fetch.
    :param file: Where to store the downloaded data.
    :param log: Logger.
    :param chunk_size: Size of download buffer.
347942
def download_http(source: str, file: Union[str, BinaryIO], log: logging.Logger,
                  chunk_size: int=DEFAULT_DOWNLOAD_CHUNK_SIZE) -> None:
    
    log.info("Fetching %s...", source)
    r = requests.get(source, stream=True)
    if r.status_code != 200:
        log.error(
            "An error occurred while fetching the model, with code %s" % r.status_code)
        raise ValueError
    if isinstance(file, str):
        os.makedirs(os.path.dirname(file), exist_ok=True)
        f = open(file, "wb")
    else:
        f = file
    try:
        total_length = int(r.headers.get("content-length"))
        num_chunks = math.ceil(total_length / chunk_size)
        if num_chunks == 1:
            f.write(r.content)
        else:
            for chunk in progress_bar(
                    r.iter_content(chunk_size=chunk_size),
                    log,
                    expected_size=num_chunks):
                if chunk:
                    f.write(chunk)
    finally:
        if isinstance(file, str):
            f.close()
1274604
def download_blob_to(file) #:doc:
        file.binmode
        blob.download { |chunk| file.write(chunk) }
        file.flush
        file.rewind
      end
912437
def download_to_file(self, local_file, headers=None, cookies=None,
                         chunk_size=1024):
        

        if not headers:
            headers = HEADERS

        if not cookies:
            cookies = {}

        req = requests.get(self.url, headers=headers, cookies=cookies,
                           stream=True)
        with open(local_file, "wb") as local_download:
            for chunk in req.iter_content(chunk_size):
                if chunk:
                    local_download.write(chunk)
319362
def retrieve_file(options)
        file_uri = options.fetch(:url)
        file_size = options.fetch(:file_size)

        retrieved = 0
        File.open(file_uri.path, 'rb') do |f|
          until f.eof?
            chunk = f.read(chunk_size)
            retrieved += chunk.length
            yield(chunk, retrieved, file_size)
          end
        end
      end
1144933
def stream(self, url, params=None, chunk_size=64*1024):
        ''
        r = self.request(url, params=params)
        for chunk in r.iter_content(chunk_size):
            yield chunk
951332
def _download(self, url, path=None, force=False):
        """"""
        if path is None:
            path = url.split('/')[-1]

        # Make dir if non existent
        folder = os.path.dirname(os.path.abspath(path))

        if not os.path.isdir(folder):
            os.makedirs(folder)

        # Start actual download
        try:
            r = requests.get(url, stream=True, proxies=self.proxy_servers)
        except Exception as error:
            print('ERROR', 'here', error)
            logger.error(str(error))
            # Break if error found!
#            self._sig_download_finished.emit(url, path)
#            return path

        total_size = int(r.headers.get('Content-Length', 0))

        # Check if file exists
        if os.path.isfile(path) and not force:
            file_size = os.path.getsize(path)

            # Check if existing file matches size of requested file
            if file_size == total_size:
                self._sig_download_finished.emit(url, path)
                return path

        # File not found or file size did not match. Download file.
        progress_size = 0
        with open(path, 'wb') as f:
            for chunk in r.iter_content(chunk_size=self._chunk_size):
                if chunk:
                    f.write(chunk)
                    progress_size += len(chunk)
                    self._sig_download_progress.emit(url, path,
                                                     progress_size,
                                                     total_size)
            self._sig_download_finished.emit(url, path)

        return path
267755
def http_stream(url, chunk=4096):
    
    ret = requests.get(url, stream=True)
    ret.raise_for_status()
    for data in ret.iter_content(chunk):
        yield data
1196307
def download_file(pk):
    
    release_file = models.ReleaseFile.objects.get(pk=pk)
    logging.info("Downloading %s", release_file.url)

    proxies = None
    if settings.LOCALSHOP_HTTP_PROXY:
        proxies = settings.LOCALSHOP_HTTP_PROXY
    response = requests.get(release_file.url, stream=True, proxies=proxies)

    # Write the file to the django file field
    filename = os.path.basename(release_file.url)

    # Setting the size manually since Django can't figure it our from
    # the raw HTTPResponse
    if 'content-length' in response.headers:
        size = int(response.headers['content-length'])
    else:
        size = len(response.content)

    # Setting the content type by first looking at the response header
    # and falling back to guessing it from the filename
    default_content_type = 'application/octet-stream'
    content_type = response.headers.get('content-type')
    if content_type is None or content_type == default_content_type:
        content_type = mimetypes.guess_type(filename)[0] or default_content_type

    # Using Django's temporary file upload system to not risk memory
    # overflows
    with TemporaryUploadedFile(name=filename, size=size, charset='utf-8',
                               content_type=content_type) as temp_file:
        temp_file.write(response.content)
        temp_file.seek(0)

        # Validate the md5 hash of the downloaded file
        md5_hash = md5_hash_file(temp_file)
        if md5_hash != release_file.md5_digest:
            logging.error("MD5 hash mismatch: %s (expected: %s)" % (
                md5_hash, release_file.md5_digest))
            return

        release_file.distribution.save(filename, temp_file)
        release_file.save()
    logging.info("Complete")
812630
def download_ftp_url(source_url, target_uri, buffer_size=8192):
    """"""

    ensure_file_directory(target_uri)

    with urllib.request.urlopen(source_url) as source_file:
        with open(target_uri, 'wb') as target_file:
            shutil.copyfileobj(source_file, target_file, buffer_size)
1257587
def download_file(save_path, file_url):
    

    r = requests.get(file_url)  # create HTTP response object

    with open(save_path, 'wb') as f:
        f.write(r.content)

    return save_path
1146282
def _download_file_from_drive(filename, url):  # pragma: no cover
    
    confirm_token = None

    # Since the file is big, drive will scan it for virus and take it to a
    # warning page. We find the confirm token on this page and append it to the
    # URL to start the download process.
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token

    logger.info("Downloading %s to %s" % (url, filename))

    response = session.get(url, stream=True)
    # Now begin the download.
    chunk_size = 16 * 1024
    with open(filename, "wb") as f:
        for chunk in response.iter_content(chunk_size):
            if chunk:
                f.write(chunk)

    # Print newline to clear the carriage return from the download progress
    statinfo = os.stat(filename)
    logger.info("Successfully downloaded %s, %s bytes." % (filename, statinfo.st_size))
158980
def fetch_model(self, source: str, file: Union[str, BinaryIO],
                    chunk_size: int=DEFAULT_DOWNLOAD_CHUNK_SIZE) -> None:
        """"""
        download_http(source, file, self._log, chunk_size)
1131178
def download(self, path, chunksize=None, chunk_callback=lambda _: False):
        
        chunksize = chunksize or self.chunksize
        bucket, obj = self._path_to_bucket_and_key(path)

        with tempfile.NamedTemporaryFile(delete=False) as fp:
            # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
            return_fp = _DeleteOnCloseFile(fp.name, 'r')

            # Special case empty files because chunk-based downloading doesn't work.
            result = self.client.objects().get(bucket=bucket, object=obj).execute()
            if int(result['size']) == 0:
                return return_fp

            request = self.client.objects().get_media(bucket=bucket, object=obj)
            downloader = http.MediaIoBaseDownload(fp, request, chunksize=chunksize)

            attempts = 0
            done = False
            while not done:
                error = None
                try:
                    _, done = downloader.next_chunk()
                    if chunk_callback(fp):
                        done = True
                except errors.HttpError as err:
                    error = err
                    if err.resp.status < 500:
                        raise
                    logger.warning('Error downloading file, retrying', exc_info=True)
                except RETRYABLE_ERRORS as err:
                    logger.warning('Error downloading file, retrying', exc_info=True)
                    error = err

                if error:
                    attempts += 1
                    if attempts >= NUM_RETRIES:
                        raise error
                else:
                    attempts = 0

        return return_fp
861661
def download_from_url(path, url):
  
  filename = url.split("/")[-1]
  found_file = find_file(path, filename, max_depth=0)
  if found_file is None:
    filename = os.path.join(path, filename)
    tf.logging.info("Downloading from %s to %s." % (url, filename))
    inprogress_filepath = filename + ".incomplete"
    inprogress_filepath, _ = urllib.request.urlretrieve(
        url, inprogress_filepath, reporthook=download_report_hook)
    # Print newline to clear the carriage return from the download progress.
    print()
    tf.gfile.Rename(inprogress_filepath, filename)
    return filename
  else:
    tf.logging.info("Already downloaded: %s (at %s)." % (url, found_file))
    return found_file
252255
def download_file(url, path=None, clobber=False):
    
    if path is None:
        local_filename = os.path.join(directory, url.split('/')[-1])
    else:
        local_filename = path

    if os.path.exists(local_filename) and not clobber:
        logging.info('{} exists; not downloading.'.format(local_filename))
        return local_filename
        
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)
                #f.flush() commented by recommendation from J.F.Sebastian
    return local_filename
650940
def download(self, url, file):
        
        if isinstance(file, str) or isinstance(file, unicode):
            file = open(file, 'wb')
        webclient = self.dupe()
        webclient.request('GET', url)
        rsp = webclient.getresponse()
        self._currdl = rsp
        self._dlfile = file
        for chunk in iter(lambda: rsp.read(16384), ''):
            file.write(chunk)
        self._currdl = None
        file.close()
245206
def stream_download(url, target_path, verbose=False):
    
    response = requests.get(url, stream=True)
    handle = open(target_path, "wb")
    if verbose:
        print("Beginning streaming download of %s" % url)
        start = datetime.now()
        try:
            content_length = int(response.headers['Content-Length'])
            content_MB = content_length/1048576.0
            print("Total file size: %.2f MB" % content_MB)
        except KeyError:
            pass      # allow Content-Length to be missing
    for chunk in response.iter_content(chunk_size=512):
        if chunk:     # filter out keep-alive new chunks
            handle.write(chunk)
    if verbose:
        print(
            "Download completed to %s in %s" %
            (target_path, datetime.now() - start))