Downloads a file using http get and request
@param {string} source - The http URL to download from
@param {object} options - Options object
@returns {Promise}
7460
function download(source, { verbose, output, onStart, onProgress } = {}) {
  return new Promise(function(y, n) {
    if (typeof output === 'undefined') {
      output = path.basename(url.parse(source).pathname) || 'unknown';
    }

    /**
     * Parse the source url into parts
     */
    const sourceUrl = url.parse(source);

    /**
     * Determine to use https or http request depends on source url
     */
    let request = null;
    if (sourceUrl.protocol === 'https:') {
      request = https.request;
    } else if (sourceUrl.protocol === 'http:') {
      request = http.request;
    } else {
      throw new Error('protocol should be http or https');
    }

    /**
     * Issue the request
     */
    const req = request(
      {
        method: 'GET',
        protocol: sourceUrl.protocol,
        host: sourceUrl.hostname,
        port: sourceUrl.port,
        path: sourceUrl.pathname + (sourceUrl.search || '')
      },
      function(res) {
        if (res.statusCode === 200) {
          const fileSize = Number.isInteger(res.headers['content-length'] - 0)
            ? parseInt(res.headers['content-length'])
            : 0;
          let downloadedSize = 0;

          /**
           * Create write stream
           */
          var writeStream = fs.createWriteStream(output, {
            flags: 'w+',
            encoding: 'binary'
          });

          res.pipe(writeStream);

          /**
           * Invoke `onStartCallback` function
           */
          if (onStart) {
            onStart(res.headers);
          }

          res.on('data', function(chunk) {
            downloadedSize += chunk.length;
            if (onProgress) {
              onProgress({
                fileSize,
                downloadedSize,
                percentage: fileSize > 0 ? downloadedSize / fileSize : 0
              });
            }
          });

          res.on('error', function(err) {
            writeStream.end();
            n(err);
          });

          writeStream.on('finish', function() {
            writeStream.end();
            req.end('finished');
            y({ headers: res.headers, fileSize });
          });
        } else if (
          res.statusCode === 301 ||
          res.statusCode === 302 ||
          res.statusCode === 307
        ) {
          const redirectLocation = res.headers.location;

          if (verbose) {
            console.log('node-wget-promise: Redirected to:', redirectLocation);
          }

          /**
           * Call download function recursively
           */
          download(redirectLocation, {
            output,
            onStart,
            onProgress
          })
            .then(y)
            .catch(n);
        } else {
          n('Server responded with unhandled status: ' + res.statusCode);
        }
      }
    );

    req.end('done');
    req.on('error', err => n(err));
  });
}
347942
def download_http(source: str, file: Union[str, BinaryIO], log: logging.Logger,
                  chunk_size: int=DEFAULT_DOWNLOAD_CHUNK_SIZE) -> None:
    
    log.info("Fetching %s...", source)
    r = requests.get(source, stream=True)
    if r.status_code != 200:
        log.error(
            "An error occurred while fetching the model, with code %s" % r.status_code)
        raise ValueError
    if isinstance(file, str):
        os.makedirs(os.path.dirname(file), exist_ok=True)
        f = open(file, "wb")
    else:
        f = file
    try:
        total_length = int(r.headers.get("content-length"))
        num_chunks = math.ceil(total_length / chunk_size)
        if num_chunks == 1:
            f.write(r.content)
        else:
            for chunk in progress_bar(
                    r.iter_content(chunk_size=chunk_size),
                    log,
                    expected_size=num_chunks):
                if chunk:
                    f.write(chunk)
    finally:
        if isinstance(file, str):
            f.close()
581205
protected final void downloadFile(final URI source, final File destination) throws IOException {
        Validate.validState(!isStopped, "Cannot download file when the crawler is not started.");
        Validate.validState(!isStopping, "Cannot download file when the crawler is stopping.");
        Validate.notNull(source, "The source URL cannot be null.");
        Validate.notNull(destination, "The destination file cannot be null.");

        HttpGet request = new HttpGet(source);
        try (CloseableHttpResponse response = httpClient.execute(request)) {
            HttpEntity entity = response.getEntity();
            if (entity != null) {
                FileUtils.copyInputStreamToFile(entity.getContent(), destination);
            }
        }
    }
1278370
function (source, options) {
        this.index = this._geIndex(source);
        this.data = '';
        this.options = options;
        this._cache = {};
        this._usedHeaders = [];
    }
535230
function(successCallback, errorCallback, options) {
        var source = options[0];
        var target = options[1];
        var downloadId = options[3];
        var headers = options[4] || {};

        if (!target) {
            errorCallback(new FTErr(FTErr.FILE_NOT_FOUND_ERR));
            return;
        }
        if (target.substr(0, 8) === "file:///") {
            target = appData.localFolder.path + target.substr(8).split("/").join("\\");
        } else if (target.indexOf('ms-appdata:///') === 0) {
            // Handle 'ms-appdata' scheme
            target = target.replace('ms-appdata:///local', appData.localFolder.path)
                           .replace('ms-appdata:///temp', appData.temporaryFolder.path);
        } else if (target.indexOf('cdvfile://') === 0) {
            target = target.replace('cdvfile://localhost/persistent', appData.localFolder.path)
                           .replace('cdvfile://localhost/temporary', appData.temporaryFolder.path);
        }
        target = cordovaPathToNative(target);

        var path = target.substr(0, target.lastIndexOf("\\"));
        var fileName = target.substr(target.lastIndexOf("\\") + 1);
        if (path === null || fileName === null) {
            errorCallback(new FTErr(FTErr.FILE_NOT_FOUND_ERR));
            return;
        }
        // Download to a temp file to avoid the file deletion on 304 
        // CB-7006 Empty file is created on file transfer if server response is 304
        var tempFileName = '~' + fileName;

        var download = null;

        // Create internal download operation object
        fileTransferOps[downloadId] = new FileTransferOperation(FileTransferOperation.PENDING, null);

        var downloadCallback = function(storageFolder) {
            storageFolder.createFileAsync(tempFileName, Windows.Storage.CreationCollisionOption.replaceExisting).then(function (storageFile) {

                if (alreadyCancelled(downloadId)) {
                    errorCallback(new FTErr(FTErr.ABORT_ERR, source, target));
                    return;
                }

                // if download isn't cancelled, contunue with creating and preparing download operation
                var downloader = new Windows.Networking.BackgroundTransfer.BackgroundDownloader();
                for (var header in headers) {
                    if (headers.hasOwnProperty(header)) {
                        downloader.setRequestHeader(header, headers[header]);
                    }
                }

                // create download object. This will throw an exception if URL is malformed
                try {
                    var uri = Windows.Foundation.Uri(source);
                    download = downloader.createDownload(uri, storageFile);
                } catch (e) {
                    // so we handle this and call errorCallback
                    errorCallback(new FTErr(FTErr.INVALID_URL_ERR));
                    return;
                }

                var downloadOperation = download.startAsync();
                // update internal TransferOperation object with newly created promise
                fileTransferOps[downloadId].promise = downloadOperation;

                downloadOperation.then(function () {

                    // Update TransferOperation object with new state, delete promise property
                    // since it is not actual anymore
                    var currentDownloadOp = fileTransferOps[downloadId];
                    if (currentDownloadOp) {
                        currentDownloadOp.state = FileTransferOperation.DONE;
                        currentDownloadOp.promise = null;
                    }

                    storageFile.renameAsync(fileName, Windows.Storage.CreationCollisionOption.replaceExisting).done(function () {
                        var nativeURI = storageFile.path.replace(appData.localFolder.path, 'ms-appdata:///local')
                        .replace(appData.temporaryFolder.path, 'ms-appdata:///temp')
                        .replace(/\\/g, '/');

                        // Passing null as error callback here because downloaded file should exist in any case
                        // otherwise the error callback will be hit during file creation in another place
                        FileProxy.resolveLocalFileSystemURI(successCallback, null, [nativeURI]);
                    }, function(error) {
                        errorCallback(new FTErr(FTErr.FILE_NOT_FOUND_ERR, source, target, null, null, error));
                    });
                }, function(error) {

                    var getTransferError = new WinJS.Promise(function (resolve) {
                        // Handle download error here. If download was cancelled,
                        // message property will be specified
                        if (error.message === 'Canceled') {
                            resolve(new FTErr(FTErr.ABORT_ERR, source, target, null, null, error));
                        } else if (error && error.number === HTTP_E_STATUS_NOT_MODIFIED) {
                            resolve(new FTErr(FTErr.NOT_MODIFIED_ERR, source, target, 304, null, error));
                        } else {
                            // in the other way, try to get response property
                            var response = download.getResponseInformation();
                            if (!response) {
                                resolve(new FTErr(FTErr.CONNECTION_ERR, source, target));
                            } else {
                                if (download.progress.bytesReceived === 0) {
                                    resolve(new FTErr(FTErr.FILE_NOT_FOUND_ERR, source, target, response.statusCode, null, error));
                                    return;
                                }
                                var reader = new Windows.Storage.Streams.DataReader(download.getResultStreamAt(0));
                                reader.loadAsync(download.progress.bytesReceived).then(function (bytesLoaded) {
                                    var payload = reader.readString(bytesLoaded);
                                    resolve(new FTErr(FTErr.FILE_NOT_FOUND_ERR, source, target, response.statusCode, payload, error));
                                });
                            }
                        }
                    });
                    getTransferError.then(function (fileTransferError) {

                        // Update TransferOperation object with new state, delete promise property
                        // since it is not actual anymore
                        var currentDownloadOp = fileTransferOps[downloadId];
                        if (currentDownloadOp) {
                            currentDownloadOp.state = FileTransferOperation.CANCELLED;
                            currentDownloadOp.promise = null;
                        }

                        // Cleanup, remove incompleted file
                        storageFile.deleteAsync().then(function() {
                            errorCallback(fileTransferError);
                        });
                    });

                }, function(evt) {

                    var progressEvent = new ProgressEvent('progress', {
                        loaded: evt.progress.bytesReceived,
                        total: evt.progress.totalBytesToReceive,
                        target: evt.resultFile
                    });
                    // when bytesReceived == 0, BackgroundDownloader has not yet differentiated whether it could get file length or not,
                    // when totalBytesToReceive == 0, BackgroundDownloader is unable to get file length
                    progressEvent.lengthComputable = (evt.progress.bytesReceived > 0) && (evt.progress.totalBytesToReceive > 0);

                    successCallback(progressEvent, { keepCallback: true });
                });
            }, function(error) {
                errorCallback(new FTErr(FTErr.FILE_NOT_FOUND_ERR, source, target, null, null, error));
            });
        };

        var fileNotFoundErrorCallback = function(error) {
            errorCallback(new FTErr(FTErr.FILE_NOT_FOUND_ERR, source, target, null, null, error));
        };

        Windows.Storage.StorageFolder.getFolderFromPathAsync(path).then(downloadCallback, function (error) {
            // Handle non-existent directory
            if (error.number === -2147024894) {
                var parent = path.substr(0, path.lastIndexOf('\\')),
                    folderNameToCreate = path.substr(path.lastIndexOf('\\') + 1);

                Windows.Storage.StorageFolder.getFolderFromPathAsync(parent).then(function(parentFolder) {
                    parentFolder.createFolderAsync(folderNameToCreate).then(downloadCallback, fileNotFoundErrorCallback);
                }, fileNotFoundErrorCallback);
            } else {
                fileNotFoundErrorCallback();
            }
        });
    }
306165
function fetch(filename) {
        var att = atts[filename];
        var path = encodeDocId(doc._id) + '/' + encodeAttachmentId(filename) +
          '?rev=' + doc._rev;
        return ajaxPromise(opts, {
          method: 'GET',
          url: genDBUrl(host, path),
          binary: true
        }).then(function (blob) {
          if (opts.binary) {
            return blob;
          }
          return new Promise(function (resolve) {
            pouchdbBinaryUtils.blobOrBufferToBase64(blob, resolve);
          });
        }).then(function (data) {
          delete att.stub;
          delete att.length;
          att.data = data;
        });
      }
284308
function S3StreamDownload (s3, s3Params, options) {
    var downloader = new Downloader(s3, s3Params, options);
    return new DownloadStream(downloader);
}
485719
def extract_download_options(options)
        url = options.fetch('url')

        # This avoids the potential for a KeyError
        headers = options.fetch('headers', {}) || {}

        file_size_value = options.fetch('file_size', 0)
        file_size = file_size_value.to_i

        output = {
          url: ::Addressable::URI.parse(url),
          headers: headers,
          file_size: file_size
        }

        output[:file_size] = get_file_size(output) if output[:file_size] < 1
        output
      end
656504
function copy (source, target) {
    var defer = Q.defer()
    fs.createReadStream(source)
      .on('error', defer.reject.bind(defer))
      .pipe(fs.createWriteStream(target))
      .on('error', defer.reject.bind(defer))
      .on('finish', defer.fulfill.bind(defer))

    return defer.promise
  }
709703
function download(downloadUrl, filename, callback, ondata) {
    var url = require("url").parse(downloadUrl);
    var out = require("fs").createWriteStream(filename, { flags: 'w', encoding: null, mode: 0666 });
    var bytes = 0, total = -1;
    var options = {
        "hostname" : url["host"],
        "method" : "GET",
        "path" : url["path"],
        "agent" : false
    }

    // Reconfigure request options if an HTTP proxy setting exists
    if (process.env.http_proxy) {
        var http_proxy = require("url").parse(process.env.http_proxy);
        options["hostname"] = http_proxy["hostname"];
        options["port"] = http_proxy["port"];
        options["path"] = url["href"];
        options["headers"] = {
            "Host" : url["host"]
        };
    }

    var req = require(path.join(__dirname, "..", "lib", "follow-redirects.js")).http.request(options,
                                                                                             function(res) {
        if (res.headers["content-length"]) {
            total = parseInt(res.headers["content-length"], 10);
        }
        if (res.statusCode != 200) {
            res.setEncoding(null);
            callback(new Error("Download failed: HTTP status code "+res.statusCode), -1);
            return;
        }
        res.on("data", function(chunk) {
            bytes += chunk.length;
            if (ondata) ondata(chunk.length, total);
            out.write(chunk);
        });

        res.on("end", function() {
            callback(null, bytes);
        });
    });
    req.on("error", function(e) {
        callback(e, -1);
    });
    req.end();
}
57933
function copyFile(source, target) {
    var deferred,
        rd,
        wr;

    if (typeof fs.copyFile === 'function') {
        return Q.ninvoke(fs, 'copyFile', source, target);
    }

    deferred = Q.defer();

    rd = fs.createReadStream(source);
    wr = fs.createWriteStream(target);

    rd.on('error', function (err) {
        deferred.reject(err);
    });

    wr.on('error', function (err) {
        deferred.reject(err);
    });
    wr.on('close', function () {
        deferred.resolve();
    });

    rd.pipe(wr);

    return deferred.promise;
}
1363338
function downloadFile(url, outputPath) {
	const dir = path.dirname(outputPath);
	if (!fs.existsSync(dir)) {
		fs.mkdirSync(dir);
	}
	return new Promise((resolve, reject) => {
		const file = fs.createWriteStream(outputPath);
		const sendReq = request.get(url);
		sendReq.pipe(file);

		sendReq
		.on("response", (response) => {
			if (response.statusCode !== 200) {
				fs.unlink(outputPath);
				const error = new Error("Response status code was " + response.statusCode + ".");
				console.trace(error);
				reject(error);
			}
		})
		.on("error", (error) => {
			fs.unlink(outputPath);
			console.trace(error);
			reject(error);
		});

		file
		.on("finish", () => {
			file.close(() => {
				resolve();
			});
		})
		.on("error", (error) => {
			fs.unlink(outputPath);
			console.trace(error);
			reject(error);
		});
	});
}
660765
function request(url, options, callback) {
	var debug = require('debug')('wayback:http');
	debug('req', url);

	callback = typeof callback === 'undefined' ? options : callback;

	// @see https://www.npmjs.com/package/fetch
	var stream = new fetch(url, typeof options === 'object' ? options : undefined);

	stream.on('error', function(err) {
		debug('error', err);
		callback(err, null);
	});

	stream.on('meta', function(meta) {
		debug('resp', 'HTTP ' + meta.status);
		debug('resp', meta.responseHeaders);
		callback(null, stream);
	});
}
56258
function copy(inputdir, outputdir, options) {
	const _options = Object.assign({ dir: '.' }, options);
	fsPlus.copySync(path.join(inputdir, _options.dir), outputdir);
	return Promise.resolve();
}
109710
function request(options) {
  if (!options) {
    return Promise.reject('Missing options.')
  }
  const data = JSON.stringify(options.data)
  return new Promise((resolve, reject) => {
    const req = https.request({
      host: BASE_URL,
      port: 443,
      method: options.method,
      path: options.path,
      headers: Object.assign({
        'Accept': '*/*',
        'Content-Type': 'application/json',
        'User-Agent': USER_AGENT,
        'Content-Length': data.length
      }, options.headers)
    }, (response) => {
      const body = []
      response.on('data', chunk => body.push(chunk))
      response.on('end', () => {
        let parsedBody = body.join('')

        try {
          parsedBody = JSON.parse(parsedBody)
        } catch (e) {
          debug('Received non-JSON data from API.', body)
        }

        const res = {
          statusCode: response.statusCode,
          statusMessage: response.statusMessage,
          headers: response.headers,
          body: parsedBody
        }
        if (response.statusCode < 200 || response.statusCode > 299) {
          reject(res)
        } else {
          resolve(res)
        }
      })
    })
    if (data) {
      req.write(data)
    }
    req.on('error', e => reject(e))
    req.end()
  })
}
282852
function httpGet(url) {
    switch (configuredDownloadClient) {
        case "curl": return sh`curl -A curl -s ${url}`;
        case "wget": return sh`wget -qO- ${url}`;
        case "httpie": return sh`http -b GET ${url}`;
        case "fetch": return sh`fetch -q ${url}`;
    }
}
317817
function download (data, filename) {
  const element = getElement();
  const url = getObjectUrl(data);

  element.setAttribute('href', url);
  element.setAttribute('download', filename);
  document.body.appendChild(element);
  element.click();
  document.body.removeChild(element);

  setTimeout(function () {
    URL.revokeObjectURL(url);
  }, 100);
}