Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 41b5b72

Browse files
dhermesdhermes
committed
Merge pull request googleapis#802 from dhermes/bucket-iter-combine
Combining get_all_blobs and Bucket.iterator.
2 parents 96ee9d3 + 34a025f commit 41b5b72

File tree

5 files changed

+87
-86
lines changed

5 files changed

+87
-86
lines changed

‎docs/_components/storage-getting-started.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,9 +156,9 @@ bucket object::
156156
>>> bucket = storage.get_bucket('my-bucket', connection=connection)
157157

158158
If you want to get all the blobs in the bucket, you can use
159-
:func:`get_all_blobs <gcloud.storage.bucket.Bucket.get_all_blobs>`::
159+
:func:`list_blobs <gcloud.storage.bucket.Bucket.list_blobs>`::
160160

161-
>>> blobs = bucket.get_all_blobs()
161+
>>> blobs = bucket.list_blobs()
162162

163163
However, if you're looking to iterate through the blobs, you can use the
164164
bucket itself as an iterator::

‎docs/_components/storage-quickstart.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ you can create buckets and blobs::
6767
>>> blob = blob.upload_from_string('this is test content!')
6868
>>> print blob.download_as_string()
6969
'this is test content!'
70-
>>> print bucket.get_all_blobs()
70+
>>> print bucket.list_blobs()
7171
[<Blob: my-new-bucket, my-test-file.txt>]
7272
>>> blob.delete()
7373
>>> bucket.delete()

‎gcloud/storage/bucket.py‎

Lines changed: 48 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
False
2424
2525
If you want to get all the blobs in the bucket, you can use
26-
:func:`get_all_blobs <gcloud.storage.bucket.Bucket.get_all_blobs>`::
26+
:func:`list_blobs <gcloud.storage.bucket.Bucket.list_blobs>`::
2727
28-
>>> blobs = bucket.get_all_blobs()
28+
>>> blobs = bucket.list_blobs()
2929
3030
You can also use the bucket as an iterator::
3131
@@ -104,7 +104,7 @@ def __repr__(self):
104104
return '<Bucket: %s>' % self.name
105105

106106
def __iter__(self):
107-
return iter(self._iterator_class(bucket=self))
107+
return iter(self.list_blobs())
108108

109109
def __contains__(self, blob_name):
110110
blob = Blob(blob_name, bucket=self)
@@ -223,56 +223,68 @@ def get_blob(self, blob_name):
223223
except NotFound:
224224
return None
225225

226-
def get_all_blobs(self):
227-
"""List all the blobs in this bucket.
228-
229-
This will **not** retrieve all the data for all the blobs, it
230-
will only retrieve the blob paths.
231-
232-
This is equivalent to::
233-
234-
blobs = [blob for blob in bucket]
226+
def list_blobs(self, max_results=None, page_token=None, prefix=None,
227+
delimiter=None, versions=None,
228+
projection='noAcl', fields=None):
229+
"""Return an iterator used to find blobs in the bucket.
235230
236-
:rtype: list of :class:`gcloud.storage.blob.Blob`
237-
:returns: A list of all the Blob objects in this bucket.
238-
"""
239-
return list(self)
231+
:type max_results: integer or ``NoneType``
232+
:param max_results: maximum number of blobs to return.
240233
241-
def iterator(self, prefix=None, delimiter=None, max_results=None,
242-
versions=None):
243-
"""Return an iterator used to find blobs in the bucket.
234+
:type page_token: string
235+
:param page_token: opaque marker for the next "page" of blobs. If not
236+
passed, will return the first page of blobs.
244237
245-
:type prefix: string or None
238+
:type prefix: string or ``NoneType``
246239
:param prefix: optional prefix used to filter blobs.
247240
248-
:type delimiter: string or None
241+
:type delimiter: string or ``NoneType``
249242
:param delimiter: optional delimter, used with ``prefix`` to
250243
emulate hierarchy.
251244
252-
:type max_results: integer or None
253-
:param max_results: maximum number of blobs to return.
254-
255-
:type versions: boolean or None
245+
:type versions: boolean or ``NoneType``
256246
:param versions: whether object versions should be returned as
257247
separate blobs.
258248
259-
:rtype: :class:`_BlobIterator`
249+
:type projection: string or ``NoneType``
250+
:param projection: If used, must be 'full' or 'noAcl'. Defaults to
251+
'noAcl'. Specifies the set of properties to return.
252+
253+
:type fields: string or ``NoneType``
254+
:param fields: Selector specifying which fields to include in a
255+
partial response. Must be a list of fields. For example
256+
to get a partial response with just the next page token
257+
and the language of each blob returned:
258+
'items/contentLanguage,nextPageToken'
259+
260+
:rtype: :class:`_BlobIterator`.
261+
:returns: An iterator of blobs.
260262
"""
261263
extra_params = {}
262264

265+
if max_results is not None:
266+
extra_params['maxResults'] = max_results
267+
263268
if prefix is not None:
264269
extra_params['prefix'] = prefix
265270

266271
if delimiter is not None:
267272
extra_params['delimiter'] = delimiter
268273

269-
if max_results is not None:
270-
extra_params['maxResults'] = max_results
271-
272274
if versions is not None:
273275
extra_params['versions'] = versions
274276

275-
return self._iterator_class(self, extra_params=extra_params)
277+
extra_params['projection'] = projection
278+
279+
if fields is not None:
280+
extra_params['fields'] = fields
281+
282+
result = self._iterator_class(self, extra_params=extra_params)
283+
# Page token must be handled specially since the base `Iterator`
284+
# class has it as a reserved property.
285+
if page_token is not None:
286+
result.next_page_token = page_token
287+
return result
276288

277289
def delete(self, force=False):
278290
"""Delete this bucket.
@@ -297,7 +309,7 @@ def delete(self, force=False):
297309
contains more than 256 objects / blobs.
298310
"""
299311
if force:
300-
blobs = list(self.iterator(
312+
blobs = list(self.list_blobs(
301313
max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1))
302314
if len(blobs) > self._MAX_OBJECTS_FOR_BUCKET_DELETE:
303315
message = (
@@ -325,7 +337,7 @@ def delete_blob(self, blob_name):
325337
>>> from gcloud import storage
326338
>>> connection = storage.get_connection()
327339
>>> bucket = storage.get_bucket('my-bucket', connection=connection)
328-
>>> print bucket.get_all_blobs()
340+
>>> print bucket.list_blobs()
329341
[<Blob: my-bucket, my-file.txt>]
330342
>>> bucket.delete_blob('my-file.txt')
331343
>>> try:
@@ -408,7 +420,7 @@ def upload_file(self, filename, blob_name=None):
408420
>>> connection = storage.get_connection()
409421
>>> bucket = storage.get_bucket('my-bucket', connection=connection)
410422
>>> bucket.upload_file('~/my-file.txt', 'remote-text-file.txt')
411-
>>> print bucket.get_all_blobs()
423+
>>> print bucket.list_blobs()
412424
[<Blob: my-bucket, remote-text-file.txt>]
413425
414426
If you don't provide a blob name, we will try to upload the file
@@ -418,7 +430,7 @@ def upload_file(self, filename, blob_name=None):
418430
>>> connection = storage.get_connection()
419431
>>> bucket = storage.get_bucket('my-bucket', connection=connection)
420432
>>> bucket.upload_file('~/my-file.txt')
421-
>>> print bucket.get_all_blobs()
433+
>>> print bucket.list_blobs()
422434
[<Blob: my-bucket, my-file.txt>]
423435
424436
:type filename: string
@@ -450,7 +462,7 @@ def upload_file_object(self, file_obj, blob_name=None):
450462
>>> connection = storage.get_connection()
451463
>>> bucket = storage.get_bucket('my-bucket', connection=connection)
452464
>>> bucket.upload_file(open('~/my-file.txt'), 'remote-text-file.txt')
453-
>>> print bucket.get_all_blobs()
465+
>>> print bucket.list_blobs()
454466
[<Blob: my-bucket, remote-text-file.txt>]
455467
456468
If you don't provide a blob name, we will try to upload the file
@@ -460,7 +472,7 @@ def upload_file_object(self, file_obj, blob_name=None):
460472
>>> connection = storage.get_connection()
461473
>>> bucket = storage.get_bucket('my-bucket', connection=connection)
462474
>>> bucket.upload_file(open('~/my-file.txt'))
463-
>>> print bucket.get_all_blobs()
475+
>>> print bucket.list_blobs()
464476
[<Blob: my-bucket, my-file.txt>]
465477
466478
:type file_obj: file

‎gcloud/storage/test_bucket.py‎

Lines changed: 28 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ def test___iter___empty(self):
101101
kw, = connection._requested
102102
self.assertEqual(kw['method'], 'GET')
103103
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
104-
self.assertEqual(kw['query_params'], {})
104+
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})
105105

106106
def test___iter___non_empty(self):
107107
NAME = 'name'
@@ -115,7 +115,7 @@ def test___iter___non_empty(self):
115115
kw, = connection._requested
116116
self.assertEqual(kw['method'], 'GET')
117117
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
118-
self.assertEqual(kw['query_params'], {})
118+
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})
119119

120120
def test___contains___miss(self):
121121
NAME = 'name'
@@ -269,58 +269,46 @@ def test_get_blob_hit(self):
269269
self.assertEqual(kw['method'], 'GET')
270270
self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME))
271271

272-
def test_get_all_blobs_empty(self):
272+
def test_list_blobs_defaults(self):
273273
NAME = 'name'
274274
connection = _Connection({'items': []})
275275
bucket = self._makeOne(NAME, connection)
276-
blobs = bucket.get_all_blobs()
277-
self.assertEqual(blobs, [])
278-
kw, = connection._requested
279-
self.assertEqual(kw['method'], 'GET')
280-
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
281-
self.assertEqual(kw['query_params'], {})
282-
283-
def test_get_all_blobs_non_empty(self):
284-
NAME = 'name'
285-
BLOB_NAME = 'blob-name'
286-
connection = _Connection({'items': [{'name': BLOB_NAME}]})
287-
bucket = self._makeOne(NAME, connection)
288-
blobs = bucket.get_all_blobs()
289-
blob, = blobs
290-
self.assertTrue(blob.bucket is bucket)
291-
self.assertEqual(blob.name, BLOB_NAME)
292-
kw, = connection._requested
293-
self.assertEqual(kw['method'], 'GET')
294-
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
295-
self.assertEqual(kw['query_params'], {})
296-
297-
def test_iterator_defaults(self):
298-
NAME = 'name'
299-
connection = _Connection({'items': []})
300-
bucket = self._makeOne(NAME, connection)
301-
iterator = bucket.iterator()
276+
iterator = bucket.list_blobs()
302277
blobs = list(iterator)
303278
self.assertEqual(blobs, [])
304279
kw, = connection._requested
305280
self.assertEqual(kw['method'], 'GET')
306281
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
307-
self.assertEqual(kw['query_params'], {})
282+
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})
308283

309-
def test_iterator_explicit(self):
284+
def test_list_blobs_explicit(self):
310285
NAME = 'name'
286+
MAX_RESULTS = 10
287+
PAGE_TOKEN = 'ABCD'
288+
PREFIX = 'subfolder'
289+
DELIMITER = '/'
290+
VERSIONS = True
291+
PROJECTION = 'full'
292+
FIELDS = 'items/contentLanguage,nextPageToken'
311293
EXPECTED = {
312-
'prefix': 'subfolder',
313-
'delimiter': '/',
314294
'maxResults': 10,
315-
'versions': True,
295+
'pageToken': PAGE_TOKEN,
296+
'prefix': PREFIX,
297+
'delimiter': DELIMITER,
298+
'versions': VERSIONS,
299+
'projection': PROJECTION,
300+
'fields': FIELDS,
316301
}
317302
connection = _Connection({'items': []})
318303
bucket = self._makeOne(NAME, connection)
319-
iterator = bucket.iterator(
320-
prefix='subfolder',
321-
delimiter='/',
322-
max_results=10,
323-
versions=True,
304+
iterator = bucket.list_blobs(
305+
max_results=MAX_RESULTS,
306+
page_token=PAGE_TOKEN,
307+
prefix=PREFIX,
308+
delimiter=DELIMITER,
309+
versions=VERSIONS,
310+
projection=PROJECTION,
311+
fields=FIELDS,
324312
)
325313
blobs = list(iterator)
326314
self.assertEqual(blobs, [])
@@ -1069,7 +1057,7 @@ def get_items_from_response(self, response):
10691057
self.assertEqual(kw[0]['query_params'], {'projection': 'full'})
10701058
self.assertEqual(kw[1]['method'], 'GET')
10711059
self.assertEqual(kw[1]['path'], '/b/%s/o' % NAME)
1072-
self.assertEqual(kw[1]['query_params'], {})
1060+
self.assertEqual(kw[1]['query_params'], {'projection': 'noAcl'})
10731061

10741062

10751063
class _Connection(object):

‎regression/storage.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -205,13 +205,13 @@ def tearDownClass(cls):
205205
blob.delete()
206206

207207
def test_list_files(self):
208-
all_blobs = self.bucket.get_all_blobs()
208+
all_blobs = list(self.bucket.list_blobs())
209209
self.assertEqual(len(all_blobs), len(self.FILENAMES))
210210

211211
def test_paginate_files(self):
212212
truncation_size = 1
213213
count = len(self.FILENAMES) - truncation_size
214-
iterator = self.bucket.iterator(max_results=count)
214+
iterator = self.bucket.list_blobs(max_results=count)
215215
response = iterator.get_next_page_response()
216216
blobs = list(iterator.get_items_from_response(response))
217217
self.assertEqual(len(blobs), count)
@@ -254,7 +254,7 @@ def tearDownClass(cls):
254254
blob.delete()
255255

256256
def test_root_level_w_delimiter(self):
257-
iterator = self.bucket.iterator(delimiter='/')
257+
iterator = self.bucket.list_blobs(delimiter='/')
258258
response = iterator.get_next_page_response()
259259
blobs = list(iterator.get_items_from_response(response))
260260
self.assertEqual([blob.name for blob in blobs], ['file01.txt'])
@@ -263,7 +263,7 @@ def test_root_level_w_delimiter(self):
263263
self.assertEqual(iterator.prefixes, ('parent/',))
264264

265265
def test_first_level(self):
266-
iterator = self.bucket.iterator(delimiter='/', prefix='parent/')
266+
iterator = self.bucket.list_blobs(delimiter='/', prefix='parent/')
267267
response = iterator.get_next_page_response()
268268
blobs = list(iterator.get_items_from_response(response))
269269
self.assertEqual([blob.name for blob in blobs], ['parent/file11.txt'])
@@ -272,7 +272,8 @@ def test_first_level(self):
272272
self.assertEqual(iterator.prefixes, ('parent/child/',))
273273

274274
def test_second_level(self):
275-
iterator = self.bucket.iterator(delimiter='/', prefix='parent/child/')
275+
iterator = self.bucket.list_blobs(delimiter='/',
276+
prefix='parent/child/')
276277
response = iterator.get_next_page_response()
277278
blobs = list(iterator.get_items_from_response(response))
278279
self.assertEqual([blob.name for blob in blobs],
@@ -288,8 +289,8 @@ def test_third_level(self):
288289
# of 1024 characters in the UTF-8 encoded name:
289290
# https://cloud.google.com/storage/docs/bucketnaming#objectnames
290291
# Exercise a layer deeper to illustrate this.
291-
iterator = self.bucket.iterator(delimiter='/',
292-
prefix='parent/child/grand/')
292+
iterator = self.bucket.list_blobs(delimiter='/',
293+
prefix='parent/child/grand/')
293294
response = iterator.get_next_page_response()
294295
blobs = list(iterator.get_items_from_response(response))
295296
self.assertEqual([blob.name for blob in blobs],

0 commit comments

Comments
 (0)