List S3 buckets. See also "aws s3 ls". Use "aws s3 ls NAME" to list bucket contents.
775066
def ls(args):
    
    table = []
    for bucket in filter_collection(resources.s3.buckets, args):
        bucket.LocationConstraint = clients.s3.get_bucket_location(Bucket=bucket.name)["LocationConstraint"]
        cloudwatch = resources.cloudwatch
        bucket_region = bucket.LocationConstraint or "us-east-1"
        if bucket_region != cloudwatch.meta.client.meta.region_name:
            cloudwatch = boto3.Session(region_name=bucket_region).resource("cloudwatch")
        data = get_cloudwatch_metric_stats("AWS/S3", "NumberOfObjects",
                                           start_time=datetime.utcnow() - timedelta(days=2),
                                           end_time=datetime.utcnow(), period=3600, BucketName=bucket.name,
                                           StorageType="AllStorageTypes", resource=cloudwatch)
        bucket.NumberOfObjects = int(data["Datapoints"][-1]["Average"]) if data["Datapoints"] else None
        data = get_cloudwatch_metric_stats("AWS/S3", "BucketSizeBytes",
                                           start_time=datetime.utcnow() - timedelta(days=2),
                                           end_time=datetime.utcnow(), period=3600, BucketName=bucket.name,
                                           StorageType="StandardStorage", resource=cloudwatch)
        bucket.BucketSizeBytes = format_number(data["Datapoints"][-1]["Average"]) if data["Datapoints"] else None
        table.append(bucket)
    page_output(tabulate(table, args))
41899
def nodes_aws(c_obj):
    """"""
    aws_nodes = []
    try:
        aws_nodes = c_obj.list_nodes()
    except BaseHTTPError as e:
        abort_err("\r HTTP Error with AWS: {}".format(e))
    aws_nodes = adj_nodes_aws(aws_nodes)
    return aws_nodes
1062777
def s3(ctx, bucket_name, data_file, region):
    """"""
    if not ctx.data_file:
        ctx.data_file = data_file

    if not ctx.bucket_name:
        ctx.bucket_name = bucket_name

    if not ctx.region:
        ctx.region = region

    ctx.type = 's3'
1206180
function (req, res) {
                var s3 = new AWS.S3(),
                    handleListBuckets = function (err, data) {
                        if (err) {
                            res.status(500).send(err);
                        } else {
                            // Add link
                            data.Buckets.forEach(function (bucket) {
                                bucket.links = [{
                                    rel: 'self',
                                    href: getBucketLink(req, bucket.Name)
                                }];
                            });
                            
                            res.json(data.Buckets);
                        }
                    };
                
                s3.listBuckets(handleListBuckets);
            }
1231295
def list(self, path=None, with_metadata=False, include_partitions=False):
        ''''''
        import json

        sub_path = self.prefix + '/' + path.strip('/') if path else self.prefix

        l = {}

        for e in self.bucket.list(sub_path):

            path = e.name.replace(self.prefix, '', 1).strip('/')
            if path.startswith('_') or path.startswith('meta'):
                continue

            # TODO 'include_partitions' doesn't make any sense outside of ambry
            if not include_partitions and path.count('/') > 1:
                continue  # partition files

            if with_metadata:
                d = self.metadata(path)
                if d and 'identity' in d:
                    d['identity'] = json.loads(d['identity'])
            else:
                d = {}

            d['caches'] = [self.repo_id]

            if path:
                l[path] = d

        return l
1209396
def s3_etag(url: str) -> Optional[str]:
    """"""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_object = s3_resource.Object(bucket_name, s3_path)
    return s3_object.e_tag
1151822
def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_acls):
    
    bucket['keys'] = []
    keys = handle_truncated_response(api_client.list_objects, {'Bucket': bucket_name}, ['Contents'])
    bucket['keys_count'] = len(keys['Contents'])
    key_count = 0
    update_status(key_count, bucket['keys_count'], 'keys')
    for key in keys['Contents']:
        key_count += 1
        key['name'] = key.pop('Key')
        key['LastModified'] = str(key['LastModified'])
        if check_encryption:
            try:
                # The encryption configuration is only accessible via an HTTP header, only returned when requesting one object at a time...
                k = api_client.get_object(Bucket = bucket_name, Key = key['name'])
                key['ServerSideEncryption'] = k['ServerSideEncryption'] if 'ServerSideEncryption' in k else None
                key['SSEKMSKeyId'] = k['SSEKMSKeyId'] if 'SSEKMSKeyId' in k else None
            except Exception as e:
                printException(e)
                continue
        if check_acls:
            try:
                key['grantees'] = get_s3_acls(api_client, bucket_name, bucket, key_name = key['name'])
            except Exception as e:
                continue
        # Save it
        bucket['keys'].append(key)
        update_status(key_count, bucket['keys_count'], 'keys')
1194294
def file_list(load):
    '''
    
    '''
    if 'env' in load:
        # "env" is not supported; Use "saltenv".
        load.pop('env')

    ret = []

    if 'saltenv' not in load:
        return ret

    saltenv = load['saltenv']
    metadata = _init()

    if not metadata or saltenv not in metadata:
        return ret
    for bucket in _find_files(metadata[saltenv]):
        for buckets in six.itervalues(bucket):
            files = [f for f in buckets if not fs.is_file_ignored(__opts__, f)]
            ret += _trim_env_off_path(files, saltenv)

    return ret
710705
def bucket(things, key):
    """"""
    ret = defaultdict(list)
    for thing in things:
        ret[key(thing)].append(thing)
    return ret
10051
function (prefix, callback, marker, contents) {

			var search = {
				Prefix: prefix, 
				Bucket: options.bucket
			};

			if (marker) {
				search.Marker = marker;
			}

			s3.listObjects(search, function (err, list) { 

				if (!err) {

					var objects = (contents) ? contents.concat(list.Contents) : list.Contents;

					if (list.IsTruncated) {
						var new_marker = _.last(list.Contents).Key;
						listObjects(prefix, callback, new_marker, objects);
					}
					else {
						callback(_.uniq(objects, function (o) { return o.Key; }));
					}
				}
				else {
					grunt.fatal('Failed to list content of bucket ' + options.bucket + '\n' + err);
				}
			});
		}
1137213
def _storage_list(args, _):
  
  target = args['object'] if args['object'] else args['bucket']
  project = args['project']
  if target is None:
    return _storage_list_buckets(project, '*')  # List all buckets.

  bucket_name, key = datalab.storage._bucket.parse_name(target)
  if bucket_name is None:
    raise Exception('Cannot list %s; not a valid bucket name' % target)

  if key or not re.search('\?|\*|\[', target):
    # List the contents of the bucket
    if not key:
      key = '*'
    if project:
      # Only list if the bucket is in the project
      for bucket in datalab.storage.Buckets(project_id=project):
        if bucket.name == bucket_name:
          break
      else:
        raise Exception('%s does not exist in project %s' % (target, project))
    else:
      bucket = datalab.storage.Bucket(bucket_name)

    if bucket.exists():
      return _storage_list_keys(bucket, key)
    else:
      raise Exception('Bucket %s does not exist' % target)

  else:
    # Treat the bucket name as a pattern and show matches. We don't use bucket_name as that
    # can strip off wildchars and so we need to strip off gs:// here.
    return _storage_list_buckets(project, target[5:])
1009583
def _gcs_list(args, _):
  
  target = args['objects']
  project = args['project']
  if target is None:
    return _gcs_list_buckets(project, '*')  # List all buckets.

  bucket_name, key = google.datalab.storage._bucket.parse_name(target)
  if bucket_name is None:
    raise Exception('Cannot list %s; not a valid bucket name' % target)

  # If a target was specified, list keys inside it
  if target:
    if not re.search('\?|\*|\[', target):
      # If no wild characters are present in the key string, append a '/*' suffix to show all keys
      key = key.strip('/') + '/*' if key else '*'

    if project:
      # Only list if the bucket is in the project
      for bucket in google.datalab.storage.Buckets(_make_context(project)):
        if bucket.name == bucket_name:
          break
      else:
        raise Exception('%s does not exist in project %s' % (target, project))
    else:
      bucket = google.datalab.storage.Bucket(bucket_name)

    if bucket.exists():
      return _gcs_list_keys(bucket, key)
    else:
      raise Exception('Bucket %s does not exist' % target)

  else:
    # Treat the bucket name as a pattern and show matches. We don't use bucket_name as that
    # can strip off wildchars and so we need to strip off gs:// here.
    return _gcs_list_buckets(project, target.strip('/')[5:])
777443
def _storage_list_buckets(project, pattern):
  
  data = [{'Bucket': 'gs://' + bucket.name, 'Created': bucket.metadata.created_on}
          for bucket in datalab.storage.Buckets(project_id=project)
          if fnmatch.fnmatch(bucket.name, pattern)]
  return datalab.utils.commands.render_dictionary(data, ['Bucket', 'Created'])
86905
def create_bucket(self, name):
        
        log.info('Creating bucket %s...' % name)
        self.s3.create_bucket(name)
903884
def _gcs_list_buckets(project, pattern):
  
  data = [{'Bucket': 'gs://' + bucket.name, 'Created': bucket.metadata.created_on}
          for bucket in google.datalab.storage.Buckets(_make_context(project))
          if fnmatch.fnmatch(bucket.name, pattern)]
  return google.datalab.utils.commands.render_dictionary(data, ['Bucket', 'Created'])
1205613
def _storage_list_keys(bucket, pattern):
  
  data = [{'Name': item.metadata.name,
           'Type': item.metadata.content_type,
           'Size': item.metadata.size,
           'Updated': item.metadata.updated_on}
          for item in _storage_get_keys(bucket, pattern)]
  return datalab.utils.commands.render_dictionary(data, ['Name', 'Type', 'Size', 'Updated'])
755212
def list_buckets(self, instance):
        
        # Server does not do pagination on listings of this resource.
        # Return an iterator anyway for similarity with other API methods
        response = self._client.get_proto(path='/buckets/' + instance)
        message = rest_pb2.ListBucketsResponse()
        message.ParseFromString(response.content)
        buckets = getattr(message, 'bucket')
        return iter([
            Bucket(bucket, instance, self) for bucket in buckets])