Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b48ce7a

Browse files
dhermesdhermes
committed
Implementing storage regression tests to match gcloud-node.
Note meant to be merged. Some missing features were uncovered or hard to use APIs and I want to discuss here and then map out the plan (i.e. how to break this up).
1 parent abe3d7c commit b48ce7a

13 files changed

+315
-22
lines changed

‎CONTRIBUTING.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,10 @@ Running Regression Tests
160160
so you'll need to provide some environment variables to facilitate
161161
authentication to your project:
162162

163+
- ``GCLOUD_TESTS_PROJECT_ID``: Developers Console project ID (e.g.
164+
bamboo-shift-455).
163165
- ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to.
166+
This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``.
164167
- ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're
165168
authenticating with
166169
- ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file.

‎gcloud/storage/bucket.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,8 @@ def upload_file(self, filename, key=None):
284284
if key is None:
285285
key = os.path.basename(filename)
286286
key = self.new_key(key)
287-
return key.set_contents_from_filename(filename)
287+
key.upload_from_filename(filename)
288+
return key
288289

289290
def upload_file_object(self, file_obj, key=None):
290291
"""Shortcut method to upload a file object into this bucket.

‎gcloud/storage/iterator.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,12 @@ class Iterator(object):
3838
:type path: string
3939
:param path: The path to query for the list of items.
4040
"""
41-
def __init__(self, connection, path):
41+
def __init__(self, connection, path, extra_params=None):
4242
self.connection = connection
4343
self.path = path
4444
self.page_number = 0
4545
self.next_page_token = None
46+
self.extra_params = extra_params
4647

4748
def __iter__(self):
4849
"""Iterate through the list of items."""
@@ -69,8 +70,13 @@ def get_query_params(self):
6970
:rtype: dict or None
7071
:returns: A dictionary of query parameters or None if there are none.
7172
"""
73+
result = None
7274
if self.next_page_token:
73-
return {'pageToken': self.next_page_token}
75+
result = {'pageToken': self.next_page_token}
76+
if self.extra_params is not None:
77+
result = result or {}
78+
result.update(self.extra_params)
79+
return result
7480

7581
def get_next_page_response(self):
7682
"""Requests the next page from the path provided.

‎gcloud/storage/key.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -337,9 +337,9 @@ def upload_from_string(self, data, content_type='text/plain'):
337337
"""
338338
string_buffer = StringIO()
339339
string_buffer.write(data)
340-
self.set_contents_from_file(file_obj=string_buffer, rewind=True,
341-
size=string_buffer.len,
342-
content_type=content_type)
340+
self.upload_from_file(file_obj=string_buffer, rewind=True,
341+
size=string_buffer.len,
342+
content_type=content_type)
343343
return self
344344

345345
# NOTE: Alias for boto-like API.
@@ -452,10 +452,11 @@ class _KeyIterator(Iterator):
452452
:type bucket: :class:`gcloud.storage.bucket.Bucket`
453453
:param bucket: The bucket from which to list keys.
454454
"""
455-
def __init__(self, bucket):
455+
def __init__(self, bucket, extra_params=None):
456456
self.bucket = bucket
457457
super(_KeyIterator, self).__init__(
458-
connection=bucket.connection, path=bucket.path + '/o')
458+
connection=bucket.connection, path=bucket.path + '/o',
459+
extra_params=extra_params)
459460

460461
def get_items_from_response(self, response):
461462
"""Factory method, yields :class:`.storage.key.Key` items from response.

‎gcloud/storage/test_bucket.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,8 +334,9 @@ def __init__(self, bucket, name):
334334
self._bucket = bucket
335335
self._name = name
336336

337-
def set_contents_from_filename(self, filename):
337+
def upload_from_filename(self, filename):
338338
_uploaded.append((self._bucket, self._name, filename))
339+
339340
bucket = self._makeOne()
340341
with _Monkey(MUT, Key=_Key):
341342
bucket.upload_file(FILENAME)
@@ -354,8 +355,9 @@ def __init__(self, bucket, name):
354355
self._bucket = bucket
355356
self._name = name
356357

357-
def set_contents_from_filename(self, filename):
358+
def upload_from_filename(self, filename):
358359
_uploaded.append((self._bucket, self._name, filename))
360+
359361
bucket = self._makeOne()
360362
with _Monkey(MUT, Key=_Key):
361363
bucket.upload_file(FILENAME, KEY)

‎gcloud/storage/test_iterator.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,13 @@ def test_get_query_params_w_token(self):
7575
self.assertEqual(iterator.get_query_params(),
7676
{'pageToken': TOKEN})
7777

78+
def test_get_query_params_extra_params(self):
79+
connection = _Connection()
80+
PATH = '/foo'
81+
extra_params = {'key': 'val'}
82+
iterator = self._makeOne(connection, PATH, extra_params=extra_params)
83+
self.assertEqual(iterator.get_query_params(), extra_params)
84+
7885
def test_get_next_page_response_new_no_token_in_response(self):
7986
PATH = '/foo'
8087
TOKEN = 'token'

‎regression/data/CloudPlatform_128px_Retina.png‎

9.36 KB
Loading

‎regression/data/five-mb-file.zip‎

5 MB
Binary file not shown.

‎regression/local_test_setup.sample‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1-
export GCLOUD_TESTS_DATASET_ID="my-dataset"
1+
export GCLOUD_TESTS_PROJECT_ID="my-project"
2+
export GCLOUD_TESTS_DATASET_ID=${GCLOUD_TESTS_PROJECT_ID}
23
export GCLOUD_TESTS_CLIENT_EMAIL="[email protected]"
34
export GCLOUD_TESTS_KEY_FILE="path.key"

‎regression/regression_utils.py‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,37 +2,58 @@
22
import sys
33

44
from gcloud import datastore
5+
from gcloud import storage
56

67

78
# Defaults from shell environ. May be None.
9+
PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID')
810
DATASET_ID = os.getenv('GCLOUD_TESTS_DATASET_ID')
911
CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL')
1012
KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE')
11-
DATASETS = {}
13+
CACHED_RETURN_VALS = {}
1214

1315
ENVIRON_ERROR_MSG = """\
1416
To run the regression tests, you need to set some environment variables.
1517
Please check the Contributing guide for instructions.
1618
"""
1719

1820

19-
def get_environ():
20-
if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
21-
print >> sys.stderr, ENVIRON_ERROR_MSG
22-
sys.exit(1)
21+
def get_environ(require_datastore=False, require_storage=False):
22+
if require_datastore:
23+
if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
24+
print >> sys.stderr, ENVIRON_ERROR_MSG
25+
sys.exit(1)
26+
27+
if require_storage:
28+
if PROJECT_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
29+
print >> sys.stderr, ENVIRON_ERROR_MSG
30+
sys.exit(1)
2331

2432
return {
33+
'project_id': PROJECT_ID,
2534
'dataset_id': DATASET_ID,
2635
'client_email': CLIENT_EMAIL,
2736
'key_filename': KEY_FILENAME,
2837
}
2938

3039

3140
def get_dataset():
32-
environ = get_environ()
41+
environ = get_environ(require_datastore=True)
3342
get_dataset_args = (environ['dataset_id'], environ['client_email'],
3443
environ['key_filename'])
35-
if get_dataset_args not in DATASETS:
44+
key = ('get_dataset', get_dataset_args)
45+
if key not in CACHED_RETURN_VALS:
46+
# Cache return value for the environment.
47+
CACHED_RETURN_VALS[key] = datastore.get_dataset(*get_dataset_args)
48+
return CACHED_RETURN_VALS[key]
49+
50+
51+
def get_storage_connection():
52+
environ = get_environ(require_storage=True)
53+
get_connection_args = (environ['project_id'], environ['client_email'],
54+
environ['key_filename'])
55+
key = ('get_storage_connection', get_connection_args)
56+
if key not in CACHED_RETURN_VALS:
3657
# Cache return value for the environment.
37-
DATASETS[get_dataset_args] = datastore.get_dataset(*get_dataset_args)
38-
return DATASETS[get_dataset_args]
58+
CACHED_RETURN_VALS[key] = storage.get_connection(*get_connection_args)
59+
return CACHED_RETURN_VALS[key]

0 commit comments

Comments
 (0)