Provide potentially streaming download from S3 using gof3r
        or the AWS CLI.
766259
def download(cls, filename, input_dir, dl_dir=None):
        
        file_info = cls.parse_remote(filename)
        if not dl_dir:
            dl_dir = os.path.join(input_dir, file_info.bucket,
                                  os.path.dirname(file_info.key))
            utils.safe_makedir(dl_dir)

        out_file = os.path.join(dl_dir, os.path.basename(file_info.key))

        if not utils.file_exists(out_file):
            with file_transaction({}, out_file) as tx_out_file:
                command, prog = cls._download_cl(filename)
                if prog == "gof3r":
                    command.extend(["-p", tx_out_file])
                elif prog == "awscli":
                    command.extend([tx_out_file])
                else:
                    raise NotImplementedError(
                        "Unexpected download program %s" % prog)
                subprocess.check_call(command)
        return out_file
1153415
def _serve_asset(self, path, gzipped_asset_bytes, request):
    """"""
    mimetype = mimetypes.guess_type(path)[0] or 'application/octet-stream'
    return http_util.Respond(
        request, gzipped_asset_bytes, mimetype, content_encoding='gzip')
1273998
def download_from(self, buff, remote_path):
        
        urn = Urn(remote_path)
        if self.is_dir(urn.path()):
            raise OptionNotValid(name='remote_path', value=remote_path)

        if not self.check(urn.path()):
            raise RemoteResourceNotFound(urn.path())

        response = self.execute_request(action='download', path=urn.quote())
        buff.write(response.content)
186728
def amazon_s3(self, scene, bands):
        

        sat = self.scene_interpreter(scene)

        # Always grab MTL.txt and QA band if bands are specified
        if 'BQA' not in bands:
            bands.append('QA')

        if 'MTL' not in bands:
            bands.append('MTL')

        urls = []

        for band in bands:
            # get url for the band
            url = self.amazon_s3_url(sat, band)

            # make sure it exist
            self.remote_file_exists(url)
            urls.append(url)

        # create folder
        path = check_create_folder(join(self.download_dir, scene))

        self.output('Source: AWS S3', normal=True, arrow=True)
        for url in urls:
            self.fetch(url, path)

        return path
190287
function (req, res) {
                var s3 = new AWS.S3(),
                    bucket = req.params.name,
                    location = getLocation(req),
                    params = {
                        Bucket: bucket,
                        Delimiter: req.query.delimiter,
                        Marker: req.query.marker,
                        MaxKeys: req.query.maxKeys || DEFAULT_MAX_KEYS,
                        Prefix: req.query.prefix
                    },
                    handleListObjects = function (err, data) {
                        if (err) {
                            res.status(500).send(err);
                        } else {
                              
                            // Get link for this set
                            data.links = [{
                                rel: 'self',
                                href: getOriginalink(req)
                            }];
                            
                            // Map through bucket contents to filter out self reference and add links
                            data.Contents = data.Contents.map(function (content) {
                                // Add s3g link to reference the resource
                                content.links = [{
                                    ref: 'self',
                                    href: getS3ObjectLink(req, bucket, content)
                                }];
                                
                                // Filter out any references to the parent file
                                if (location.origin + location.search !== content.links[0].href) {
                                    return content;
                                }
                                
                            }).filter(Boolean);
                            
                            
                            // If this set of objects has a next marker that means there are > 1000 so provide the link to the next set of data
                            if (data.IsTruncated) {
                                // Get link for next set using next marker if available or the last key in the content set
                                data.links.push({
                                    rel: 'next',
                                    href: getNextMarkerLink(req, data.NextMarker || data.Contents[data.Contents.length - 1].Key)
                                });
                            }
                            res.json(data);
                        }
                    };
                
                s3.listObjects(params, handleListObjects);
            }
573925
def download(progress=True):
    ''' 

    '''
    data_dir = external_data_dir(create=True)
    print("Using data directory: %s" % data_dir)

    s3 = 'https://bokeh-sampledata.s3.amazonaws.com'
    files = [
        (s3, 'CGM.csv'),
        (s3, 'US_Counties.zip'),
        (s3, 'us_cities.json'),
        (s3, 'unemployment09.csv'),
        (s3, 'AAPL.csv'),
        (s3, 'FB.csv'),
        (s3, 'GOOG.csv'),
        (s3, 'IBM.csv'),
        (s3, 'MSFT.csv'),
        (s3, 'WPP2012_SA_DB03_POPULATION_QUINQUENNIAL.zip'),
        (s3, 'gapminder_fertility.csv'),
        (s3, 'gapminder_population.csv'),
        (s3, 'gapminder_life_expectancy.csv'),
        (s3, 'gapminder_regions.csv'),
        (s3, 'world_cities.zip'),
        (s3, 'airports.json'),
        (s3, 'movies.db.zip'),
        (s3, 'airports.csv'),
        (s3, 'routes.csv'),
        (s3, 'haarcascade_frontalface_default.xml'),
    ]

    for base_url, filename in files:
        _download_file(base_url, filename, data_dir, progress=progress)
718339
def get_file_from_s3(job, s3_url, encryption_key=None, write_to_jobstore=True):
    
    work_dir = job.fileStore.getLocalTempDir()
    filename = '/'.join([work_dir, os.path.basename(s3_url)])
    # This is common to encrypted and unencrypted downloads
    download_call = ['curl', '-fs', '--retry', '5']
    # If an encryption key was provided, use it to create teh headers that need to be injected into
    # the curl script and append to the call
    if encryption_key:
        key = generate_unique_key(encryption_key, s3_url)
        encoded_key = base64.b64encode(key)
        encoded_key_md5 = base64.b64encode( hashlib.md5(key).digest() )
        h1 = 'x-amz-server-side-encryption-customer-algorithm:AES256'
        h2 = 'x-amz-server-side-encryption-customer-key:{}'.format(encoded_key)
        h3 = 'x-amz-server-side-encryption-customer-key-md5:{}'.format(encoded_key_md5)
        download_call.extend(['-H', h1, '-H', h2, '-H', h3])
    # This is also common to both types of downloads
    download_call.extend([s3_url, '-o', filename])
    try:
        subprocess.check_call(download_call)
    except subprocess.CalledProcessError:
        raise RuntimeError('Curl returned a non-zero exit status processing %s. Do you' % s3_url +
                           'have premssions to access the file?')
    except OSError:
        raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
    assert os.path.exists(filename)
    if write_to_jobstore:
        filename = job.fileStore.writeGlobalFile(filename)
    return filename
335822
def download_from_url(url, path):
    """"""
    def process_response(r):
        chunk_size = 16 * 1024
        total_size = int(r.headers.get('Content-length', 0))
        with open(path, "wb") as file:
            with tqdm(total=total_size, unit='B',
                      unit_scale=1, desc=path.split('/')[-1]) as t:
                for chunk in r.iter_content(chunk_size):
                    if chunk:
                        file.write(chunk)
                        t.update(len(chunk))

    if 'drive.google.com' not in url:
        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, stream=True)
        process_response(response)
        return

    print('downloading from Google Drive; may take a few minutes')
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token
        response = session.get(url, stream=True)

    process_response(response)
830075
def s3_download(url, dst):  # type: (str, str) -> None
    
    url = parse.urlparse(url)

    if url.scheme != 's3':
        raise ValueError("Expecting 's3' scheme, got: %s in %s" % (url.scheme, url))

    bucket, key = url.netloc, url.path.lstrip('/')

    region = os.environ.get('AWS_REGION', os.environ.get(_params.REGION_NAME_ENV))
    s3 = boto3.resource('s3', region_name=region)

    s3.Bucket(bucket).download_file(key, dst)
977888
def generate_s3_url(files):
    """"""
    if files:
        vault = g.client.Vault.get_personal_vault()
        files = json.loads(files)
        objects = []
        for i in xrange(len(files)):
            obj = g.client.Object.create(
                vault_id=vault.id,
                object_type='file',
                filename=files[i].get('filename'),
                mimetype=files[i].get('mimetype'),
                size=files[i].get('size')
            )
            objects.append({
                'id': obj.id,
                'filename': obj.filename,
                'upload_url': obj.upload_url
            })
        return json.dumps(objects)
1308191
function processFile(bucket, key, transform) {
  // get the csv file s3://${bucket}/${key}
  console.log(`Processing s3://${bucket}/${key}`)
  const s3 = new AWS.S3()
  const csvStream = csv.parse({ headers: true, objectMode: true })
  s3.getObject({Bucket: bucket, Key: key}).createReadStream().pipe(csvStream)
  return es.streamToEs(csvStream, transform, esClient, index)
}
899501
def s3_request(func: Callable):
    

    @wraps(func)
    def wrapper(url: str, *args, **kwargs):
        try:
            return func(url, *args, **kwargs)
        except ClientError as exc:
            if int(exc.response["Error"]["Code"]) == 404:
                raise FileNotFoundError("file {} not found".format(url))
            else:
                raise

    return wrapper
261504
def download_from_url(source, destination, progress=False, uncompress=False):
    """"""
    # Modules #
    from tqdm import tqdm
    import requests
    from autopaths.file_path import FilePath
    # Check destination exists #
    destination = FilePath(destination)
    destination.directory.create_if_not_exists()
    # Over HTTP #
    response = requests.get(source, stream=True)
    total_size = int(response.headers.get('content-length'))
    block_size = int(total_size/1024)
    # Do it #
    with open(destination, "wb") as handle:
        if progress:
            for data in tqdm(response.iter_content(chunk_size=block_size), total=1024): handle.write(data)
        else:
            for data in response.iter_content(chunk_size=block_size): handle.write(data)
    # Uncompress #
    if uncompress:
        with open(destination) as f: header = f.read(4)
        if header == "PK\x03\x04": unzip(destination, inplace=True)
        # Add other compression formats here
    # Return #
    return destination
329564
def download(self, source, destination):
        """"""
        self._handler.download(posix_path(source), destination)
1217339
def fetch_s3_package(self, config):
        
        extractor_map = {'.tar.gz': TarGzipExtractor,
                         '.tar': TarExtractor,
                         '.zip': ZipExtractor}
        extractor = None
        for suffix, klass in extractor_map.items():
            if config['key'].endswith(suffix):
                extractor = klass()
                logger.debug("Using extractor %s for S3 object \"%s\" in "
                             "bucket %s.",
                             klass.__name__,
                             config['key'],
                             config['bucket'])
                dir_name = self.sanitize_uri_path(
                    "s3-%s-%s" % (config['bucket'],
                                  config['key'][:-len(suffix)])
                )
                break

        if extractor is None:
            raise ValueError(
                "Archive type could not be determined for S3 object \"%s\" "
                "in bucket %s." % (config['key'], config['bucket'])
            )

        session = get_session(region=None)
        extra_s3_args = {}
        if config.get('requester_pays', False):
            extra_s3_args['RequestPayer'] = 'requester'

        # We can skip downloading the archive if it's already been cached
        if config.get('use_latest', True):
            try:
                # LastModified should always be returned in UTC, but it doesn't
                # hurt to explicitly convert it to UTC again just in case
                modified_date = session.client('s3').head_object(
                    Bucket=config['bucket'],
                    Key=config['key'],
                    **extra_s3_args
                )['LastModified'].astimezone(dateutil.tz.tzutc())
            except botocore.exceptions.ClientError as client_error:
                logger.error("Error checking modified date of "
                             "s3://%s/%s : %s",
                             config['bucket'],
                             config['key'],
                             client_error)
                sys.exit(1)
            dir_name += "-%s" % modified_date.strftime(self.ISO8601_FORMAT)
        cached_dir_path = os.path.join(self.package_cache_dir, dir_name)
        if not os.path.isdir(cached_dir_path):
            logger.debug("Remote package s3://%s/%s does not appear to have "
                         "been previously downloaded - starting download and "
                         "extraction to %s",
                         config['bucket'],
                         config['key'],
                         cached_dir_path)
            tmp_dir = tempfile.mkdtemp(prefix='stacker')
            tmp_package_path = os.path.join(tmp_dir, dir_name)
            try:
                extractor.set_archive(os.path.join(tmp_dir, dir_name))
                logger.debug("Starting remote package download from S3 to %s "
                             "with extra S3 options \"%s\"",
                             extractor.archive,
                             str(extra_s3_args))
                session.resource('s3').Bucket(config['bucket']).download_file(
                    config['key'],
                    extractor.archive,
                    ExtraArgs=extra_s3_args
                )
                logger.debug("Download complete; extracting downloaded "
                             "package to %s",
                             tmp_package_path)
                extractor.extract(tmp_package_path)
                logger.debug("Moving extracted package directory %s to the "
                             "Stacker cache at %s",
                             dir_name,
                             self.package_cache_dir)
                shutil.move(tmp_package_path, self.package_cache_dir)
            finally:
                shutil.rmtree(tmp_dir)
        else:
            logger.debug("Remote package s3://%s/%s appears to have "
                         "been previously downloaded to %s -- bypassing "
                         "download",
                         config['bucket'],
                         config['key'],
                         cached_dir_path)

        # Update sys.path & merge in remote configs (if necessary)
        self.update_paths_and_config(config=config,
                                     pkg_dir_name=dir_name)
1240298
def upload_file_to_s3(awsclient, bucket, key, filename):
    
    client_s3 = awsclient.get_client('s3')
    transfer = S3Transfer(client_s3)
    # Upload /tmp/myfile to s3://bucket/key and print upload progress.
    transfer.upload_file(filename, bucket, key)
    response = client_s3.head_object(Bucket=bucket, Key=key)
    etag = response.get('ETag')
    version_id = response.get('VersionId', None)
    return etag, version_id
1069068
def _download_py3(link, path, __hdr__):
    """"""
    try:
        req = urllib.request.Request(link, headers=__hdr__)
        u = urllib.request.urlopen(req)
    except Exception as e:
        raise Exception(' Download failed with the error:\n{}'.format(e))

    with open(path, 'wb') as outf:
        for l in u:
            outf.write(l)
    u.close()