Download URL to a directory.
    Will figure out the filename automatically from URL, if not given.
341669
def download(url, dir, filename=None, expect_size=None):
    
    mkdir_p(dir)
    if filename is None:
        filename = url.split('/')[-1]
    fpath = os.path.join(dir, filename)

    if os.path.isfile(fpath):
        if expect_size is not None and os.stat(fpath).st_size == expect_size:
            logger.info("File {} exists! Skip download.".format(filename))
            return fpath
        else:
            logger.warn("File {} exists. Will overwrite with a new download!".format(filename))

    def hook(t):
        last_b = [0]

        def inner(b, bsize, tsize=None):
            if tsize is not None:
                t.total = tsize
            t.update((b - last_b[0]) * bsize)
            last_b[0] = b
        return inner
    try:
        with tqdm.tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t:
            fpath, _ = urllib.request.urlretrieve(url, fpath, reporthook=hook(t))
        statinfo = os.stat(fpath)
        size = statinfo.st_size
    except IOError:
        logger.error("Failed to download {}".format(url))
        raise
    assert size > 0, "Downloaded an empty file from {}!".format(url)

    if expect_size is not None and size != expect_size:
        logger.error("File downloaded from {} does not match the expected size!".format(url))
        logger.error("You may have downloaded a broken file, or the upstream may have modified the file.")

    # TODO human-readable size
    logger.info('Succesfully downloaded ' + filename + ". " + str(size) + ' bytes.')
    return fpath
630541
def download_and_unpack(self, url, *paths, **kw):
        
        with self.temp_download(url, 'ds.zip', log=kw.pop('log', None)) as zipp:
            with TemporaryDirectory() as tmpdir:
                with zipfile.ZipFile(zipp.as_posix()) as zipf:
                    for path in paths:
                        zipf.extract(as_posix(path), path=tmpdir.as_posix())
                        copy(tmpdir.joinpath(path), self)
252255
def download_file(url, path=None, clobber=False):
    
    if path is None:
        local_filename = os.path.join(directory, url.split('/')[-1])
    else:
        local_filename = path

    if os.path.exists(local_filename) and not clobber:
        logging.info('{} exists; not downloading.'.format(local_filename))
        return local_filename
        
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)
                #f.flush() commented by recommendation from J.F.Sebastian
    return local_filename
560233
def download(url: str, filename: str,
             skip_cert_verify: bool = True) -> None:
    
    log.info("Downloading from {} to {}", url, filename)

    # urllib.request.urlretrieve(url, filename)
    # ... sometimes fails (e.g. downloading
    # https://www.openssl.org/source/openssl-1.1.0g.tar.gz under Windows) with:
    # ssl.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:777)  # noqa
    # ... due to this certificate root problem (probably because OpenSSL
    #     [used by Python] doesn't play entirely by the same rules as others?):
    # https://stackoverflow.com/questions/27804710
    # So:

    ctx = ssl.create_default_context()  # type: ssl.SSLContext
    if skip_cert_verify:
        log.debug("Skipping SSL certificate check for " + url)
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
    with urllib.request.urlopen(url, context=ctx) as u, open(filename,
                                                             'wb') as f:  # noqa
        f.write(u.read())
1220856
def _downloadfile(self, url, fname):
        '''  '''

        print("The file %s need to be download - Wait\n " %
              (fname.split('/')[-1]))
        urllib.urlretrieve(url, fname, self._report)
        print("\n The download of the file %s has succeded \n " %
              (fname.split('/')[-1]))
390182
def download_url(url, back_off=True, **kwargs):
    
    if back_off:
        return _download_with_backoff(url, as_file=True, **kwargs)
    else:
        return _download_without_backoff(url, as_file=True, **kwargs)
121883
function downloader(url, saveAtPath, callback) {
  // Handle the parameter errors
  if (!url || !saveAtPath) {
    return callback(new Error("You need to provide the URL and the location to save the file!"));
  }
  if (url && typeof url !== "string") {
    return callback(new Error("The parameter 'url' must be a string"));
  }
  if (saveAtPath && typeof saveAtPath !== "string") {
    return callback(new Error("The parameter 'saveAtPath' must be a string"));
  }
  if (callback && typeof callback !== "function") {
    throw new Error("The callback provided must be a function");
  }

  var httpClient = url.indexOf("https://") === 0 ? https : http;

  // The directory to save the file.
  var dirToSave = path.dirname(saveAtPath);

  // Verify if the directory to save exists before downloading.
  fs.stat(dirToSave, function gettingStats(err) {
    if (err) {
      return callback(new Error("The path '" + dirToSave + "' does NOT exist!"));
    }

    var req = httpClient.get(url, function requestHandler(res) {
      if (res.statusCode === 404) {
        return callback(new Error("The resource at the given URL " + url + " does NOT exist"));
      }

      var chunks = [];

      res.on("data", function(chunk) {
        chunks.push(chunk);
      });

      res.on("end", function() {
        var buffer = Buffer.concat(chunks);

        // Finished downloading and getting the buffer to be saved.
        fs.writeFile(saveAtPath, buffer, function(err) {
          if (err) {
            var error = new Error("Error while saving image: " + err.message);
            if (err.code === "EISDIR") {
              error = new Error("Cannot save the url " + url + " as a directory " + saveAtPath);
            }
            callback(error);
          }

          fs.stat(saveAtPath, function(err, stat) {
            if (err) {
              callback(new Error("Error while getting image stats: " + err.message));
            }

            // Creating the response object.
            var response = {
              downloaded: true,
              filePath: saveAtPath,
              size: stat.size
            };

            // The request to the URL was successful and the file was saved.
            callback(null, response);
          });
        });
      });

    });

    // Handle errors during the request.
    req.on("error", function(error) {
      if (error) {
        if (error.code === "ENOTFOUND") {
          return callback(new Error("There's no internet connectivity to " + url));

        } else {
          var unknownError = new Error("Unknown error to save " + url + " as " + saveAtPath);
          unknownError.stack = error.stack;
          return callback(unknownError);
        }
      }
    });
  });
}
363158
def download_file(url, file_name):
    

    logger.info("Downloading URL: %s", url)
    file_size = 0

    if not os.path.isfile(file_name):
        response = requests.get(url, stream=True)

        with open(file_name, "wb") as fp:
            if not response.ok:
                raise Exception("Download exception. Will fail.")

            for block in response.iter_content(1024):
                if not block:
                    break

                fp.write(block)
                file_size += len(block)

        logger.info("Download finished, size is %d bytes.", file_size)

    return file_size
1081604
def download(url, filename=None):
	
	# requirements os, shutil, urllib.parse, urllib.request
	if not filename:
		url_parts = urllib.parse.urlparse(url)
		filename = os.path.basename(url_parts.path)
	url_h = urllib.request.urlopen(url)
	with open(filename, 'wb') as file_h:
		shutil.copyfileobj(url_h, file_h)
	url_h.close()
	return
858194
def download_file(filename: str):
    

    project = cd.project.get_internal_project()
    source_directory = project.source_directory if project else None

    if not filename or not project or not source_directory:
        return '', 204

    path = os.path.realpath(os.path.join(
        source_directory,
        '..',
        '__cauldron_downloads',
        filename
    ))

    if not os.path.exists(path):
        return '', 204

    return flask.send_file(path, mimetype=mimetypes.guess_type(path)[0])
1078451
def download_url(self, url, **kwargs):
        
        if self.baseurl and '://' not in url:
            url = join(self.baseurl, url)
        return self.resolver.download_to_directory(self.directory, url, **kwargs)
180863
def _get_url(self, url):
        
        url = str(url)
        if url != 'default' and not '://' in url:
            url = "file:" + urllib.pathname2url(url)
        return url
60691
def download_file_from_url(self, source_app, url):
        
        if source_app:
            source_name = source_app
        else:
            source_name = urlparse.urlparse(url).netloc.replace('.', '_')

        filename = self.create_file_name(source_name)
        self.download_file(url, filename)
        return filename
807378
def download_file(local_filename, url, clobber=False):
    """"""
    dir_name = os.path.dirname(local_filename)
    mkdirs(dir_name)

    if clobber or not os.path.exists(local_filename):
        i = requests.get(url)

        # if not exists
        if i.status_code == 404:
            print('Failed to download file:', local_filename, url)
            return False

        # write out in 1MB chunks
        chunk_size_in_bytes = 1024*1024  # 1MB
        with open(local_filename, 'wb') as local_file:
            for chunk in i.iter_content(chunk_size=chunk_size_in_bytes):
                local_file.write(chunk)

    return True
995934
def download(url, localFileName=None, localDirName=None):
    
    localName = url2name(url)
    req = Request(url)
    r = urlopen(req)
    if r.info().has_key('Content-Disposition'):
        # If the response has Content-Disposition, we take file name from it
        localName = r.info()['Content-Disposition'].split('filename=')
        if len(localName) > 1:
            localName = localName[1]
            if localName[0] == '"' or localName[0] == "'":
                localName = localName[1:-1]
        else:
            localName = url2name(r.url)
    elif r.url != url:
        # if we were redirected, the real file name we take from the final URL
        localName = url2name(r.url)
    if localFileName:
        # we can force to save the file as specified name
        localName = localFileName
    if localDirName:
        # we can also put it in some custom directory
        if not os.path.exists(localDirName):
            os.makedirs(localDirName)
        localName = os.path.join(localDirName, localName)

    f = open(localName, 'wb')
    f.write(r.read())
    f.close()
1205451
def _get_url(self, url):
        
        url = str(url)
        if url != 'default' and not '://' in url:
            url = ':///'.join(('file', url))
        return url
5268
def download_url(url, dir_name='.', save_name=None, store_directory=None, messages=True, suffix=''):
    """"""
    if sys.version_info>=(3,0):
        from urllib.parse import quote
        from urllib.request import urlopen 
        from urllib.error import HTTPError, URLError
    else:
        from urllib2 import quote
        from urllib2 import urlopen
        from urllib2 import URLError as HTTPError
    i = url.rfind('/')
    file = url[i+1:]
    if store_directory is not None:
        dir_name = os.path.join(dir_name, store_directory)
    if save_name is None:
        save_name = file
    save_name = os.path.join(dir_name, save_name)
    print("Downloading ", url, "->", save_name)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    try:
        response = urlopen(url+suffix)
    except HTTPError as e:
        if not hasattr(e, "code"):
            raise
        if e.code > 399 and e.code<500:
            raise ValueError('Tried url ' + url + suffix + ' and received client error ' + str(e.code))
        elif e.code > 499:
            raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(e.code))
    except URLError as e:
        raise ValueError('Tried url ' + url + suffix + ' and failed with error ' + str(e.reason))
    with open(save_name, 'wb') as f:
        meta = response.info()
        content_length_str = meta.get("Content-Length")
        if content_length_str:
            #if sys.version_info>=(3,0):
            try:
                file_size = int(content_length_str)
            except:
                try: 
                    file_size = int(content_length_str[0])
                except:
                    file_size = None
            if file_size == 1:
                file_size = None
            #else:
            #    file_size = int(content_length_str)
        else:
            file_size = None
        
        status = ""
        file_size_dl = 0
        block_sz = 8192
        line_length = 30
        percentage = 1./line_length
        
        if file_size:
            print("|"+"{:^{ll}}".format("Downloading {:7.3f}MB".format(file_size/(1048576.)), ll=line_length)+"|")
            from itertools import cycle
            cycle_str = cycle('>')
            sys.stdout.write("|")
                    
        while True:
            buff = response.read(block_sz)
            if not buff:
                break
            file_size_dl += len(buff)
            f.write(buff)
            
            # If content_length_str was incorrect, we can end up with many too many equals signs, catches this edge case
            #correct_meta = float(file_size_dl)/file_size <= 1.0
            
            if file_size:
                if (float(file_size_dl)/file_size) >= percentage:
                    sys.stdout.write(next(cycle_str))
                    sys.stdout.flush()
                    percentage += 1./line_length
                #percentage = "="*int(line_length*float(file_size_dl)/file_size)
                #status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1048576.), full=file_size/(1048576.), ll=line_length, perc=percentage)
            else:
                sys.stdout.write(" "*(len(status)) + "\r")
                status = r"{dl:7.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       ll=line_length,
                                                                       perc="."*int(line_length*float(file_size_dl/(10*1048576.))))
                sys.stdout.write(status)
                sys.stdout.flush()

            #sys.stdout.write(status)
        
        if file_size:
            sys.stdout.write("|")
            sys.stdout.flush()

        print(status)