Download a file from NCBI Gene's ftp server.
359380
def dnld_ncbi_gene_file(fin, force_dnld=False, log=sys.stdout, loading_bar=True):
    """"""
    if not os.path.exists(fin) or force_dnld:
        import gzip
        fin_dir, fin_base = os.path.split(fin)
        fin_gz = "{F}.gz".format(F=fin_base)
        fin_gz = os.path.join(fin_dir, fin_gz)
        if os.path.exists(fin_gz):
            os.remove(fin_gz)
        fin_ftp = "ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/{F}.gz".format(F=fin_base)
        ## if log is not None:
        ##     log.write("  DOWNLOADING GZIP: {GZ}\n".format(GZ=fin_ftp))
        ## if loading_bar:
        ##     loading_bar = wget.bar_adaptive
        ## wget.download(fin_ftp, bar=loading_bar)
        ## rsp = wget(fin_ftp)
        ftp_get(fin_ftp, fin_gz)
        with gzip.open(fin_gz, 'rb') as zstrm:
            if log is not None:
                log.write("\n  READ GZIP:  {F}\n".format(F=fin_gz))
            with open(fin, 'wb') as ostrm:
                ostrm.write(zstrm.read())
                if log is not None:
                    log.write("  WROTE UNZIPPED: {F}\n".format(F=fin))
870940
def read_blob(arg):
    """"""
    result = None
    if arg == '@-':
        result = sys.stdin.read()
    elif any(arg.startswith('@{}://'.format(x)) for x in {'http', 'https', 'ftp', 'file'}):
        if not requests:
            raise error.UserError("You must 'pip install requests' to support @URL arguments.")
        try:
            response = requests.get(arg[1:])
            response.raise_for_status()
            result = response.content
        except requests.RequestException as exc:
            raise error.UserError(str(exc))
    else:
        with open(os.path.expanduser(arg[1:]), 'rb') as handle:
            result = handle.read()

    return result
286885
def download_url(url, filename, headers):
    """"""
    ensure_dirs(filename)
    response = requests.get(url, headers=headers, stream=True)
    if response.status_code == 200:
        with open(filename, 'wb') as f:
            for chunk in response.iter_content(16 * 1024):
                f.write(chunk)
1105873
def download_file(url, data_path='.', filename=None, size=None, chunk_size=4096, verbose=True):
    """"""
    if filename is None:
        filename = dropbox_basename(url)
    file_path = os.path.join(data_path, filename)
    if url.endswith('?dl=0'):
        url = url[:-1] + '1'  # noninteractive download
    if verbose:
        tqdm_prog = tqdm
        print('requesting URL: {}'.format(url))
    else:
        tqdm_prog = no_tqdm
    r = requests.get(url, stream=True, allow_redirects=True, timeout=5)
    size = r.headers.get('Content-Length', None) if size is None else size
    print('remote size: {}'.format(size))

    stat = path_status(file_path)
    print('local size: {}'.format(stat.get('size', None)))
    if stat['type'] == 'file' and stat['size'] == size:  # TODO: check md5 or get the right size of remote file
        r.close()
        return file_path

    print('Downloading to {}'.format(file_path))

    with open(file_path, 'wb') as f:
        for chunk in r.iter_content(chunk_size=chunk_size):
            if chunk:  # filter out keep-alive chunks
                f.write(chunk)

    r.close()
    return file_path
179736
def download(self, FILENAME=None):
        """"""
        resultFilename = None
        url = self.__url+'/services'                
        result = Util.retrieveJsonResponseFromServer(url)
        dataItems = result['data']
        for item in dataItems:
            plugin = Plugin(item)        
            result = plugin.getParameterByValue("fr.cnes.sitools.resources.order.DirectOrderResource")            
            if result == None:
                continue
            else:
                urlParameter = plugin.getParameterByType('PARAMETER_ATTACHMENT')
                urlPlugin = urlParameter.getValue()
                encodingParameter = plugin.getParameterByName('archiveType')
                encodingPlugin = encodingParameter.getValue()
                query = self.getQueries()
                query.setBaseUrl(self.__url + urlPlugin)
                url = query.getUrl()                
                (filename, header) = Util.urlretrieve('%s' % url, FILENAME)
                if FILENAME == None:
                    os.rename(filename, filename + "." + encodingPlugin)
                    resultFilename = filename + "." + encodingPlugin
                else:
                    os.rename(FILENAME, FILENAME + "." + encodingPlugin)
                    resultFilename = FILENAME + "." + encodingPlugin
                break
        return resultFilename
972217
def download_file(self, *args, **kwargs):
        """"""
        return download_file(*args, **self._merge_overrides(**kwargs)).run()
1181129
def download(link, outdir='.', chunk_size=4096):
    '''
    
    '''
    url = None
    fh = None
    eta = 'unknown '
    bytes_so_far = 0
    filename = filename_from_url(link) or "."
    cj = cjar.CookieJar()

    # get filename for temp file in current directory
    (fd_tmp, tmpfile) = tempfile.mkstemp(
        ".tmp", prefix=filename + ".", dir=outdir)
    os.close(fd_tmp)
    os.unlink(tmpfile)

    try:
        opener = ulib.build_opener(ulib.HTTPCookieProcessor(cj))
        url = opener.open(link)
        fh = open(tmpfile, mode='wb')

        headers = url.info()
        try:
            total_size = int(headers['Content-Length'])
        except (ValueError, KeyError, TypeError):
            total_size = 'unknown'

        try:
            md5_header = headers['Content-MD5']
        except (ValueError, KeyError, TypeError):
            md5_header = None

        # Define which callback we're gonna use
        if total_size != 'unknown':
            if CONSOLE_WIDTH > 57:
                reporthook = report_bar
            else:
                reporthook = report_onlysize
        else:
            reporthook = report_unknown

        # Below are the registers to calculate network transfer rate
        time_register = time()
        speed = 0.0
        speed_list = []
        bytes_register = 0.0
        eta = 'unknown '

        # Loop that reads in chunks, calculates speed and does the callback to
        # print the progress
        while True:
            chunk = url.read(chunk_size)
            # Update Download Speed every 1 second
            if time() - time_register > 0.5:
                speed = (bytes_so_far - bytes_register) / \
                    (time() - time_register)
                speed_list.append(speed)

                # Set register properly for future use
                time_register = time()
                bytes_register = bytes_so_far

                # Estimative of remaining download time
                if total_size != 'unknown' and len(speed_list) == 3:
                    speed_mean = sum(speed_list) / 3
                    eta_sec = int((total_size - bytes_so_far) / speed_mean)
                    eta = str(datetime.timedelta(seconds=eta_sec))
                    speed_list = []

            bytes_so_far += len(chunk)

            if not chunk:
                sys.stdout.write('\n')
                break

            fh.write(chunk)
            reporthook(bytes_so_far, total_size, speed, eta)
    except KeyboardInterrupt:
        print('\n\nCtrl + C: Download aborted by user')
        print('Partial downloaded file:\n{0}'.format(os.path.abspath(tmpfile)))
        sys.exit(1)
    finally:
        if url:
            url.close()
        if fh:
            fh.close()

    filenamealt = filename_from_headers(headers)
    if filenamealt:
        filename = filenamealt

    # add numeric '(x)' suffix if filename already exists
    if os.path.exists(os.path.join(outdir, filename)):
        filename = filename_fix_existing(filename, outdir)
    filename = os.path.join(outdir, filename)

    shutil.move(tmpfile, filename)

    # Check if sizes matches
    if total_size != 'unknown' and total_size != bytes_so_far:
        print(
            '\n\nWARNING!! Downloaded file size mismatches... Probably corrupted...')

    # Check md5 if it was in html header
    if md5_header:
        print('\nValidating MD5 checksum...')
        if md5_header == md5sum(filename):
            print('MD5 checksum passed!')
        else:
            print('MD5 checksum do NOT passed!!!')

    return filename
569219
def fetch(self, is_dl_forced=False):
        
        self.get_files(is_dl_forced)

        ncbi = NCBIGene(self.graph_type, self.are_bnodes_skized)
        # ncbi.fetch()
        gene_group = ncbi.files['gene_group']
        self.fetch_from_url(
            gene_group['url'], '/'.join((ncbi.rawdir, gene_group['file'])), False)

        # load and tag a list of OMIM IDs with types
        # side effect of populating omim replaced
        self.omim_type = self.find_omim_type()

        return
1121853
def download_kegg_gene_metadata(gene_id, outdir=None, force_rerun=False):
    
    if not outdir:
        outdir = ''

    # Replace colon with dash in the KEGG gene ID
    outfile = op.join(outdir, '{}.kegg'.format(custom_slugify(gene_id)))

    if ssbio.utils.force_rerun(flag=force_rerun, outfile=outfile):
        raw_text = bs_kegg.get("{}".format(gene_id))
        if raw_text == 404:
            return

        with io.open(outfile, mode='wt', encoding='utf-8') as f:
            f.write(raw_text)

        log.debug('{}: downloaded KEGG metadata file'.format(outfile))
    else:
        log.debug('{}: KEGG metadata file already exists'.format(outfile))

    return outfile
966556
def download_file(self, url):
        
        response = requests.get(url, stream=True)
        response.raise_for_status()
        return (int(response.headers.get('content-length', 0)), response)
1058771
private void downloadProtein() throws IOException, InterruptedException {
        logger.info("Downloading protein information ...");
        Path proteinFolder = common.resolve("protein");

        if (!Files.exists(proteinFolder)) {
            makeDir(proteinFolder);
            String url = configuration.getDownload().getUniprot().getHost();
            downloadFile(url, proteinFolder.resolve("uniprot_sprot.xml.gz").toString());
            String relNotesUrl = configuration.getDownload().getUniprotRelNotes().getHost();
            downloadFile(relNotesUrl, proteinFolder.resolve("uniprotRelnotes.txt").toString());
            saveVersionData(EtlCommons.PROTEIN_DATA, UNIPROT_NAME, getLine(proteinFolder.resolve("uniprotRelnotes.txt"), 1),
                    getTimeStamp(), Collections.singletonList(url), proteinFolder.resolve("uniprotVersion.json"));

            makeDir(proteinFolder.resolve("uniprot_chunks"));
            splitUniprot(proteinFolder.resolve("uniprot_sprot.xml.gz"), proteinFolder.resolve("uniprot_chunks"));

            url = configuration.getDownload().getIntact().getHost();
            downloadFile(url, proteinFolder.resolve("intact.txt").toString());
            saveVersionData(EtlCommons.PROTEIN_DATA, INTACT_NAME, null, getTimeStamp(), Collections.singletonList(url),
                    proteinFolder.resolve("intactVersion.json"));

            url = configuration.getDownload().getInterpro().getHost();
            downloadFile(url, proteinFolder.resolve("protein2ipr.dat.gz").toString());
            relNotesUrl = configuration.getDownload().getInterproRelNotes().getHost();
            downloadFile(relNotesUrl, proteinFolder.resolve("interproRelnotes.txt").toString());
            saveVersionData(EtlCommons.PROTEIN_DATA, INTERPRO_NAME, getLine(proteinFolder.resolve("interproRelnotes.txt"), 5),
                    getTimeStamp(), Collections.singletonList(url), proteinFolder.resolve("interproVersion.json"));

        } else {
            logger.info("Protein: skipping this since it is already downloaded. Delete 'protein' folder to force download");
        }
    }
1065468
def download():
    

    ftp = ftplib.FTP(SITE)
    ftp.set_debuglevel(DEBUG)
    ftp.login(USER, PASSWD)
    ftp.cwd(DIR)
    filelist = ftp.nlst()
    filecounter = MANAGER.counter(total=len(filelist), desc='Downloading',
                                  unit='files')

    for filename in filelist:

        with Writer(filename, ftp.size(filename), DEST) as writer:
            ftp.retrbinary('RETR %s' % filename, writer.write)
        print(filename)
        filecounter.update()

    ftp.close()
367759
function download(url, hostHeader, destinationFilename, callback) {
	if (typeof destinationFilename === 'function') {
		callback = destinationFilename;
		destinationFilename = null;
	}

	let options = require('url').parse(url);
	options.method = "GET";
	options.headers = {
		"Host": hostHeader,
		"Accept": "text/html,*/*;q=0.9",
		"Accept-Encoding": "gzip,identity,*;q=0",
		"Accept-Charset": "ISO-8859-1,utf-8,*;q=0.7",
		"User-Agent": "Valve/Steam HTTP Client 1.0"
	};

	let req = require('http').request(options, (res) => {
		if (res.statusCode != 200) {
			callback(new Error("HTTP error " + res.statusCode));
			return;
		}

		res.setEncoding('binary'); // apparently using null just doesn't work... thanks node
		let stream = res;

		if (res.headers['content-encoding'] && res.headers['content-encoding'] == 'gzip') {
			stream = require('zlib').createGunzip();
			stream.setEncoding('binary');
			res.pipe(stream);
		}

		let totalSizeBytes = parseInt(res.headers['content-length'] || 0, 10);
		let receivedBytes = 0;
		let dataBuffer = new Buffer(0);

		if (destinationFilename) {
			stream.pipe(require('fs').createWriteStream(destinationFilename));
		}

		stream.on('data', (chunk) => {
			if (typeof chunk === 'string') {
				chunk = new Buffer(chunk, 'binary');
			}

			receivedBytes += chunk.length;

			if (!destinationFilename) {
				dataBuffer = Buffer.concat([dataBuffer, chunk]);
			}

			callback(null, {"type": "progress", "receivedBytes": receivedBytes, "totalSizeBytes": totalSizeBytes});
		});

		stream.on('end', () => {
			callback(null, {"type": "complete", "data": dataBuffer});
		});
	});

	req.on('error', (err) => {
		callback(err);
	});

	req.end();
}
1211217
def maybe_download(url, work_directory):
    """"""
    filename = url.split("/")[-1]
    filepath = os.path.join(work_directory, filename)
    if not os.path.exists(filepath):
        logger.info("Downloading to {}...".format(filepath))
        download(url, work_directory)
    return filepath
328437
def fetch(
            self,
            url,
            filename=None,
            decompress=False,
            force=False,
            timeout=None,
            use_wget_if_available=True):
        
        key = (url, decompress)
        if not force and key in self._local_paths:
            path = self._local_paths[key]
            if exists(path):
                return path
            else:
                del self._local_paths[key]
        path = download.fetch_file(
            url,
            filename=filename,
            decompress=decompress,
            subdir=self.subdir,
            force=force,
            timeout=timeout,
            use_wget_if_available=use_wget_if_available)

        self._local_paths[key] = path
        return path
1018563
def downLoadFile( dbid, rid, fid, vid = "0" )

      @dbid, @rid, @fid, @vid = dbid, rid, fid, vid

      @downLoadFileURL = "http://#{@org}.#{@domain}.com/up/#{dbid}/a/r#{rid}/e#{fid}/v#{vid}"
      
      if @useSSL
         @downLoadFileURL.gsub!( "http:", "https:" )
      end

      @requestHeaders = { "Cookie"  => "ticket=#{@ticket}" }

      if @printRequestsAndResponses
         puts
         puts "downLoadFile request: -------------------------------------"
         p @downLoadFileURL
         p @requestHeaders
      end

      begin

         if USING_HTTPCLIENT
            @responseCode = 404
            @fileContents = @httpConnection.get_content( @downLoadFileURL, nil, @requestHeaders ) 
            @responseCode = 200 if @fileContents
         else  
            @responseCode, @fileContents = @httpConnection.get( @downLoadFileURL, @requestHeaders )
         end
    
      rescue Net::HTTPBadResponse => @lastError
      rescue Net::HTTPHeaderSyntaxError => @lastError
      rescue StandardError => @lastError
      end

      if @printRequestsAndResponses
         puts
         puts "downLoadFile response: -------------------------------------"
         p @responseCode
         p @fileContents
      end

      return self if @chainAPIcalls
      return @responseCode, @fileContents
    end
146682
def DownloadFile(file_obj, target_path, buffer_size=BUFFER_SIZE):
  
  logging.info(u"Downloading: %s to: %s", file_obj.urn, target_path)

  target_file = open(target_path, "wb")
  file_obj.Seek(0)
  count = 0

  data_buffer = file_obj.Read(buffer_size)
  while data_buffer:
    target_file.write(data_buffer)
    data_buffer = file_obj.Read(buffer_size)
    count += 1
    if not count % 3:
      logging.debug(u"Downloading: %s: %s done", file_obj.urn,
                    utils.FormatNumberAsString(count * buffer_size))
  target_file.close()