List all storage buckets that match a pattern.
777443
def _storage_list_buckets(project, pattern):
  
  data = [{'Bucket': 'gs://' + bucket.name, 'Created': bucket.metadata.created_on}
          for bucket in datalab.storage.Buckets(project_id=project)
          if fnmatch.fnmatch(bucket.name, pattern)]
  return datalab.utils.commands.render_dictionary(data, ['Bucket', 'Created'])
1362322
def get_bucket(self, hash_name, bucket_key):
        
        if hash_name in self.buckets:
            if bucket_key in self.buckets[hash_name]:
                return self.buckets[hash_name][bucket_key]
        return []
1160858
def _storage_get_key_names(bucket, pattern):
  
  return [item.metadata.name for item in _storage_get_keys(bucket, pattern)]
136149
def list_bucket(self, bucket):
    
    self.response.write('Listbucket result:\n')

    page_size = 1
    stats = gcs.listbucket(bucket + '/foo', max_keys=page_size)
    while True:
      count = 0
      for stat in stats:
        count += 1
        self.response.write(repr(stat))
        self.response.write('\n')

      if count != page_size or count == 0:
        break
      stats = gcs.listbucket(bucket + '/foo', max_keys=page_size,
                             marker=stat.filename)
1005775
public static <T> List<List<T>> getBuckets(Collection<Pair<T, Long>> files, double bucketHigh, double bucketLow, long minSSTableSize)
    {
        // Sort the list in order to get deterministic results during the grouping below
        List<Pair<T, Long>> sortedFiles = new ArrayList<Pair<T, Long>>(files);
        Collections.sort(sortedFiles, new Comparator<Pair<T, Long>>()
        {
            public int compare(Pair<T, Long> p1, Pair<T, Long> p2)
            {
                return p1.right.compareTo(p2.right);
            }
        });

        Map<Long, List<T>> buckets = new HashMap<Long, List<T>>();

        outer:
        for (Pair<T, Long> pair: sortedFiles)
        {
            long size = pair.right;

            // look for a bucket containing similar-sized files:
            // group in the same bucket if it's w/in 50% of the average for this bucket,
            // or this file and the bucket are all considered "small" (less than `minSSTableSize`)
            for (Entry<Long, List<T>> entry : buckets.entrySet())
            {
                List<T> bucket = entry.getValue();
                long oldAverageSize = entry.getKey();
                if ((size > (oldAverageSize * bucketLow) && size < (oldAverageSize * bucketHigh))
                    || (size < minSSTableSize && oldAverageSize < minSSTableSize))
                {
                    // remove and re-add under new new average size
                    buckets.remove(oldAverageSize);
                    long totalSize = bucket.size() * oldAverageSize;
                    long newAverageSize = (totalSize + size) / (bucket.size() + 1);
                    bucket.add(pair.left);
                    buckets.put(newAverageSize, bucket);
                    continue outer;
                }
            }

            // no similar bucket found; put it in a new one
            ArrayList<T> bucket = new ArrayList<T>();
            bucket.add(pair.left);
            buckets.put(size, bucket);
        }

        return new ArrayList<List<T>>(buckets.values());
    }
719222
def listdir(self, name):
        
        return [], [obj.filename for obj in cloudstorage.listbucket(self.path(name))]
670802
def _find_files(metadata):
    '''
    
    '''

    ret = {}

    for bucket, data in six.iteritems(metadata):
        if bucket not in ret:
            ret[bucket] = []

        # grab the paths from the metadata
        filePaths = [k['Key'] for k in data]
        # filter out the dirs
        ret[bucket] += [k for k in filePaths if not k.endswith('/')]

    return ret
1205613
def _storage_list_keys(bucket, pattern):
  
  data = [{'Name': item.metadata.name,
           'Type': item.metadata.content_type,
           'Size': item.metadata.size,
           'Updated': item.metadata.updated_on}
          for item in _storage_get_keys(bucket, pattern)]
  return datalab.utils.commands.render_dictionary(data, ['Name', 'Type', 'Size', 'Updated'])
979070
def _get_candidates(self, v):
        
        candidates = []
        for lshash in self.lshashes:
            for bucket_key in lshash.hash_vector(v, querying=True):
                bucket_content = self.storage.get_bucket(
                    lshash.hash_name,
                    bucket_key,
                )
                #print 'Bucket %s size %d' % (bucket_key, len(bucket_content))
                candidates.extend(bucket_content)
        return candidates
370454
def data_names(self):
        """"""
        if self.binded:
            return self._curr_module.data_names
        else:
            _, data_names, _ = self._call_sym_gen(self._default_bucket_key)
            return data_names
999830
def finddirs(pattern, path='.', exclude=None, recursive=True):
    """"""
    import fnmatch
    import os
    if recursive:
        for root, dirnames, filenames in os.walk(path):
            for pat in _to_list(pattern):
                for dirname in fnmatch.filter(dirnames, pat):
                    dirpath = join(abspath(root), dirname)
                    for excl in _to_list(exclude):
                        if excl and fnmatch.fnmatch(dirpath, excl):
                            break
                    else:
                        yield dirpath
    else:
        for pat in _to_list(pattern):
            for dirname in fnmatch.filter(listdirs(path), pat):
                dirpath = join(abspath(path), dirname)
                for excl in _to_list(exclude):
                    if excl and fnmatch.fnmatch(dirpath, excl):
                        break
                else:
                    yield dirpath
347707
def contains_glob(path_to_directory, pattern, wanted=None):
    """"""
    if not path_to_directory:
        return False
    needed = make_needed(pattern, path_to_directory, wanted)
    for name in _names_in_directory(path_to_directory):
        if needed(name):
            return True
    return False
124607
private Set<String> getBucketsForPath(String path) {

        Set<String> result = Sets.newHashSet();
        boolean foundBucket = false;
        for (int i = 0; i < m_bucketNames.size(); i++) {
            for (String bucketPath : m_bucketPathLists.get(i)) {
                if (CmsStringUtil.isPrefixPath(bucketPath, path)) {
                    String bucketName = m_bucketNames.get(i);
                    result.add(bucketName);
                    if (!BUCKET_OTHER.equals(bucketName)) {
                        foundBucket = true;
                    }
                }
            }
        }
        if (!foundBucket) {
            result.add(BUCKET_OTHER);
        }
        return result;
    }
1245013
def list(cls, path):
        
        file_info = cls.parse_remote(path)
        connection = cls.connect(path)
        bucket = connection.get_bucket(file_info.bucket)
        region = "@%s" % file_info.region if file_info.region else ""

        output = []
        for key in bucket.get_all_keys(prefix=file_info.key):
            output.append(cls._S3_FILE % {"bucket": file_info.bucket,
                                          "key": key.name,
                                          "region": region})
        return output
1231295
def list(self, path=None, with_metadata=False, include_partitions=False):
        ''''''
        import json

        sub_path = self.prefix + '/' + path.strip('/') if path else self.prefix

        l = {}

        for e in self.bucket.list(sub_path):

            path = e.name.replace(self.prefix, '', 1).strip('/')
            if path.startswith('_') or path.startswith('meta'):
                continue

            # TODO 'include_partitions' doesn't make any sense outside of ambry
            if not include_partitions and path.count('/') > 1:
                continue  # partition files

            if with_metadata:
                d = self.metadata(path)
                if d and 'identity' in d:
                    d['identity'] = json.loads(d['identity'])
            else:
                d = {}

            d['caches'] = [self.repo_id]

            if path:
                l[path] = d

        return l
230873
def _find_file_meta(metadata, bucket_name, saltenv, path):
    '''
    
    '''
    env_meta = metadata[saltenv] if saltenv in metadata else {}
    bucket_meta = {}
    for bucket in env_meta:
        if bucket_name in bucket:
            bucket_meta = bucket[bucket_name]
    files_meta = list(list(filter((lambda k: 'Key' in k), bucket_meta)))

    for item_meta in files_meta:
        if 'Key' in item_meta and item_meta['Key'] == path:
            try:
                # Get rid of quotes surrounding md5
                item_meta['ETag'] = item_meta['ETag'].strip('"')
            except KeyError:
                pass
            return item_meta
782400
def list(self, dataPath, ext=None, start=None, stop=None, recursive=False):
        
        scheme, bucket_name, keylist = self.getfiles(
            dataPath, ext=ext, start=start, stop=stop, recursive=recursive)

        return ["%s:///%s/%s" % (scheme, bucket_name, key) for key in keylist]