diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f94e9a878d54..b588d1ec8c97 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -160,7 +160,10 @@ Running Regression Tests so you'll need to provide some environment variables to facilitate authentication to your project: + - ``GCLOUD_TESTS_PROJECT_ID``: Developers Console project ID (e.g. + bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. + This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're authenticating with - ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file. diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 72b10ec93359..e9e3d012377c 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -276,7 +276,8 @@ def upload_file(self, filename, key=None): if key is None: key = os.path.basename(filename) key = self.new_key(key) - return key.set_contents_from_filename(filename) + key.upload_from_filename(filename) + return key def upload_file_object(self, file_obj, key=None): """Shortcut method to upload a file object into this bucket. diff --git a/gcloud/storage/iterator.py b/gcloud/storage/iterator.py index 9f3d098df4cf..b6e2e3512641 100644 --- a/gcloud/storage/iterator.py +++ b/gcloud/storage/iterator.py @@ -38,11 +38,12 @@ class Iterator(object): :type path: string :param path: The path to query for the list of items. """ - def __init__(self, connection, path): + def __init__(self, connection, path, extra_params=None): self.connection = connection self.path = path self.page_number = 0 self.next_page_token = None + self.extra_params = extra_params def __iter__(self): """Iterate through the list of items.""" @@ -69,8 +70,13 @@ def get_query_params(self): :rtype: dict or None :returns: A dictionary of query parameters or None if there are none. """ + result = None if self.next_page_token: - return {'pageToken': self.next_page_token} + result = {'pageToken': self.next_page_token} + if self.extra_params is not None: + result = result or {} + result.update(self.extra_params) + return result def get_next_page_response(self): """Requests the next page from the path provided. diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 217f98ee29e3..4b7547fe7eba 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -298,9 +298,9 @@ def upload_from_string(self, data, content_type='text/plain'): """ string_buffer = StringIO() string_buffer.write(data) - self.set_contents_from_file(file_obj=string_buffer, rewind=True, - size=string_buffer.len, - content_type=content_type) + self.upload_from_file(file_obj=string_buffer, rewind=True, + size=string_buffer.len, + content_type=content_type) return self # NOTE: Alias for boto-like API. @@ -465,10 +465,11 @@ class _KeyIterator(Iterator): :type bucket: :class:`gcloud.storage.bucket.Bucket` :param bucket: The bucket from which to list keys. """ - def __init__(self, bucket): + def __init__(self, bucket, extra_params=None): self.bucket = bucket super(_KeyIterator, self).__init__( - connection=bucket.connection, path=bucket.path + '/o') + connection=bucket.connection, path=bucket.path + '/o', + extra_params=extra_params) def get_items_from_response(self, response): """Factory method, yields :class:`.storage.key.Key` items from response. diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index f471b9bb9467..24a2f3556843 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -334,8 +334,9 @@ def __init__(self, bucket, name): self._bucket = bucket self._name = name - def set_contents_from_filename(self, filename): + def upload_from_filename(self, filename): _uploaded.append((self._bucket, self._name, filename)) + bucket = self._makeOne() with _Monkey(MUT, Key=_Key): bucket.upload_file(FILENAME) @@ -354,8 +355,9 @@ def __init__(self, bucket, name): self._bucket = bucket self._name = name - def set_contents_from_filename(self, filename): + def upload_from_filename(self, filename): _uploaded.append((self._bucket, self._name, filename)) + bucket = self._makeOne() with _Monkey(MUT, Key=_Key): bucket.upload_file(FILENAME, KEY) diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py index 954bc46f1473..7ebfd7dc5a90 100644 --- a/gcloud/storage/test_iterator.py +++ b/gcloud/storage/test_iterator.py @@ -75,6 +75,13 @@ def test_get_query_params_w_token(self): self.assertEqual(iterator.get_query_params(), {'pageToken': TOKEN}) + def test_get_query_params_extra_params(self): + connection = _Connection() + PATH = '/foo' + extra_params = {'key': 'val'} + iterator = self._makeOne(connection, PATH, extra_params=extra_params) + self.assertEqual(iterator.get_query_params(), extra_params) + def test_get_next_page_response_new_no_token_in_response(self): PATH = '/foo' TOKEN = 'token' diff --git a/regression/data/CloudPlatform_128px_Retina.png b/regression/data/CloudPlatform_128px_Retina.png new file mode 100644 index 000000000000..86c04e4b44f4 Binary files /dev/null and b/regression/data/CloudPlatform_128px_Retina.png differ diff --git a/regression/data/five-mb-file.zip b/regression/data/five-mb-file.zip new file mode 100644 index 000000000000..38da09a4f799 Binary files /dev/null and b/regression/data/five-mb-file.zip differ diff --git a/regression/local_test_setup.sample b/regression/local_test_setup.sample index 539b8a36f16c..afb399044b39 100644 --- a/regression/local_test_setup.sample +++ b/regression/local_test_setup.sample @@ -1,3 +1,4 @@ -export GCLOUD_TESTS_DATASET_ID="my-dataset" +export GCLOUD_TESTS_PROJECT_ID="my-project" +export GCLOUD_TESTS_DATASET_ID=${GCLOUD_TESTS_PROJECT_ID} export GCLOUD_TESTS_CLIENT_EMAIL="some-account@developer.gserviceaccount.com" export GCLOUD_TESTS_KEY_FILE="path.key" diff --git a/regression/regression_utils.py b/regression/regression_utils.py index 5b24ef6997cf..6255231dc785 100644 --- a/regression/regression_utils.py +++ b/regression/regression_utils.py @@ -2,13 +2,15 @@ import sys from gcloud import datastore +from gcloud import storage # Defaults from shell environ. May be None. +PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID') DATASET_ID = os.getenv('GCLOUD_TESTS_DATASET_ID') CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL') KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE') -DATASETS = {} +CACHED_RETURN_VALS = {} ENVIRON_ERROR_MSG = """\ To run the regression tests, you need to set some environment variables. @@ -16,12 +18,19 @@ """ -def get_environ(): - if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None: - print >> sys.stderr, ENVIRON_ERROR_MSG - sys.exit(1) +def get_environ(require_datastore=False, require_storage=False): + if require_datastore: + if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None: + print >> sys.stderr, ENVIRON_ERROR_MSG + sys.exit(1) + + if require_storage: + if PROJECT_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None: + print >> sys.stderr, ENVIRON_ERROR_MSG + sys.exit(1) return { + 'project_id': PROJECT_ID, 'dataset_id': DATASET_ID, 'client_email': CLIENT_EMAIL, 'key_filename': KEY_FILENAME, @@ -29,10 +38,22 @@ def get_environ(): def get_dataset(): - environ = get_environ() + environ = get_environ(require_datastore=True) get_dataset_args = (environ['dataset_id'], environ['client_email'], environ['key_filename']) - if get_dataset_args not in DATASETS: + key = ('get_dataset', get_dataset_args) + if key not in CACHED_RETURN_VALS: + # Cache return value for the environment. + CACHED_RETURN_VALS[key] = datastore.get_dataset(*get_dataset_args) + return CACHED_RETURN_VALS[key] + + +def get_storage_connection(): + environ = get_environ(require_storage=True) + get_connection_args = (environ['project_id'], environ['client_email'], + environ['key_filename']) + key = ('get_storage_connection', get_connection_args) + if key not in CACHED_RETURN_VALS: # Cache return value for the environment. - DATASETS[get_dataset_args] = datastore.get_dataset(*get_dataset_args) - return DATASETS[get_dataset_args] + CACHED_RETURN_VALS[key] = storage.get_connection(*get_connection_args) + return CACHED_RETURN_VALS[key] diff --git a/regression/run_regression.py b/regression/run_regression.py index e83cacc27184..ab0d60409da2 100644 --- a/regression/run_regression.py +++ b/regression/run_regression.py @@ -11,7 +11,7 @@ def get_parser(): parser = argparse.ArgumentParser( description='GCloud test runner against actual project.') parser.add_argument('--package', dest='package', - choices=('datastore',), + choices=('datastore', 'storage'), default='datastore', help='Package to be tested.') return parser @@ -27,7 +27,10 @@ def main(): parser = get_parser() args = parser.parse_args() # Make sure environ is set before running test. - regression_utils.get_environ() + if args.package == 'datastore': + regression_utils.get_environ(require_datastore=True) + elif args.package == 'storage': + regression_utils.get_environ(require_storage=True) test_result = run_module_tests(args.package) if not test_result.wasSuccessful(): sys.exit(1) diff --git a/regression/storage.py b/regression/storage.py new file mode 100644 index 000000000000..ff90e826d22a --- /dev/null +++ b/regression/storage.py @@ -0,0 +1,247 @@ +from Crypto.Hash import MD5 +import base64 +import httplib2 +import tempfile +import time +import unittest2 + +from gcloud import storage +# This assumes the command is being run via tox hence the +# repository root is the current directory. +from regression import regression_utils + + +HTTP = httplib2.Http() +SHARED_BUCKETS = {} + + +def setUpModule(): + if 'test_bucket' not in SHARED_BUCKETS: + connection = regression_utils.get_storage_connection() + # %d rounds milliseconds to nearest integer. + bucket_name = 'new%d' % (1000 * time.time(),) + # In the **very** rare case the bucket name is reserved, this + # fails with a ConnectionError. + SHARED_BUCKETS['test_bucket'] = connection.create_bucket(bucket_name) + + +def tearDownModule(): + for bucket in SHARED_BUCKETS.values(): + # Passing force=True also deletes all files. + bucket.delete(force=True) + + +class TestStorage(unittest2.TestCase): + + @classmethod + def setUpClass(cls): + cls.connection = regression_utils.get_storage_connection() + + +class TestStorageBuckets(TestStorage): + + def setUp(self): + self.case_buckets_to_delete = [] + + def tearDown(self): + for bucket in self.case_buckets_to_delete: + bucket.delete() + + def test_create_bucket(self): + new_bucket_name = 'a-new-bucket' + self.assertRaises(storage.exceptions.NotFoundError, + self.connection.get_bucket, new_bucket_name) + created = self.connection.create_bucket(new_bucket_name) + self.case_buckets_to_delete.append(created) + self.assertEqual(created.name, new_bucket_name) + + def test_get_buckets(self): + buckets_to_create = [ + 'new%d' % (1000 * time.time(),), + 'newer%d' % (1000 * time.time(),), + 'newest%d' % (1000 * time.time(),), + ] + created_buckets = [] + for bucket_name in buckets_to_create: + bucket = self.connection.create_bucket(bucket_name) + self.case_buckets_to_delete.append(bucket) + + # Retrieve the buckets. + all_buckets = self.connection.get_all_buckets() + created_buckets = [bucket for bucket in all_buckets + if bucket.name in buckets_to_create] + self.assertEqual(len(created_buckets), len(buckets_to_create)) + + +class TestStorageFiles(TestStorage): + + FILES = { + 'logo': { + 'path': 'regression/data/CloudPlatform_128px_Retina.png', + }, + 'big': { + 'path': 'regression/data/five-mb-file.zip', + }, + } + + @staticmethod + def _get_base64_md5hash(filename): + with open(filename, 'rb') as file_obj: + hash = MD5.new(data=file_obj.read()) + digest_bytes = hash.digest() + return base64.b64encode(digest_bytes) + + @classmethod + def setUpClass(cls): + super(TestStorageFiles, cls).setUpClass() + for file_data in cls.FILES.values(): + file_data['hash'] = cls._get_base64_md5hash(file_data['path']) + cls.bucket = SHARED_BUCKETS['test_bucket'] + + def setUp(self): + self.case_keys_to_delete = [] + + def tearDown(self): + for key in self.case_keys_to_delete: + key.delete() + + +class TestStorageWriteFiles(TestStorageFiles): + + def test_large_file_write_from_stream(self): + key = self.bucket.new_key('LargeFile') + self.assertEqual(key.metadata, {}) + + file_data = self.FILES['big'] + with open(file_data['path'], 'rb') as file_obj: + self.bucket.upload_file_object(file_obj, key=key) + self.case_keys_to_delete.append(key) + + key.reload_metadata() + self.assertEqual(key.metadata['md5Hash'], file_data['hash']) + + def test_write_metadata(self): + my_metadata = {'contentType': 'image/png'} + key = self.bucket.upload_file(self.FILES['logo']['path']) + self.case_keys_to_delete.append(key) + + # NOTE: This should not be necessary. We should be able to pass + # it in to upload_file and also to upload_from_string. + key.patch_metadata(my_metadata) + self.assertEqual(key.metadata['contentType'], + my_metadata['contentType']) + + def test_direct_write_and_read_into_file(self): + key = self.bucket.new_key('MyBuffer') + file_contents = 'Hello World' + key.upload_from_string(file_contents) + self.case_keys_to_delete.append(key) + + same_key = self.bucket.new_key('MyBuffer') + temp_filename = tempfile.mktemp() + with open(temp_filename, 'w') as file_obj: + same_key.get_contents_to_file(file_obj) + + with open(temp_filename, 'rb') as file_obj: + stored_contents = file_obj.read() + + self.assertEqual(file_contents, stored_contents) + + def test_copy_existing_file(self): + key = self.bucket.upload_file(self.FILES['logo']['path'], + key='CloudLogo') + self.case_keys_to_delete.append(key) + + new_key = self.bucket.copy_key(key, self.bucket, 'CloudLogoCopy') + self.case_keys_to_delete.append(new_key) + + base_contents = key.get_contents_as_string() + copied_contents = new_key.get_contents_as_string() + self.assertEqual(base_contents, copied_contents) + + +class TestStorageListFiles(TestStorageFiles): + + FILENAMES = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3'] + + @classmethod + def setUpClass(cls): + super(TestStorageListFiles, cls).setUpClass() + # Make sure bucket empty before beginning. + for key in cls.bucket: + key.delete() + + logo_path = cls.FILES['logo']['path'] + key = cls.bucket.upload_file(logo_path, key=cls.FILENAMES[0]) + cls.suite_keys_to_delete = [key] + + # Copy main key onto remaining in FILENAMES. + for filename in cls.FILENAMES[1:]: + new_key = cls.bucket.copy_key(key, cls.bucket, filename) + cls.suite_keys_to_delete.append(new_key) + + @classmethod + def tearDownClass(cls): + for key in cls.suite_keys_to_delete: + key.delete() + + def test_list_files(self): + all_keys = self.bucket.get_all_keys() + self.assertEqual(len(all_keys), len(self.FILENAMES)) + + def test_paginate_files(self): + truncation_size = 1 + extra_params = {'maxResults': len(self.FILENAMES) - truncation_size} + iterator = storage.key._KeyIterator(bucket=self.bucket, + extra_params=extra_params) + response = iterator.get_next_page_response() + keys = list(iterator.get_items_from_response(response)) + self.assertEqual(len(keys), extra_params['maxResults']) + self.assertEqual(iterator.page_number, 1) + self.assertTrue(iterator.next_page_token is not None) + + response = iterator.get_next_page_response() + last_keys = list(iterator.get_items_from_response(response)) + self.assertEqual(len(last_keys), truncation_size) + + +class TestStorageSignURLs(TestStorageFiles): + + def setUp(self): + super(TestStorageSignURLs, self).setUp() + + logo_path = self.FILES['logo']['path'] + with open(logo_path, 'r') as file_obj: + self.LOCAL_FILE = file_obj.read() + + key = self.bucket.new_key('LogoToSign.jpg') + key.upload_from_string(self.LOCAL_FILE) + self.case_keys_to_delete.append(key) + + def tearDown(self): + for key in self.case_keys_to_delete: + if key.exists(): + key.delete() + + def test_create_signed_read_url(self): + key = self.bucket.new_key('LogoToSign.jpg') + expiration = int(time.time() + 5) + signed_url = key.generate_signed_url(expiration, method='GET') + + response, content = HTTP.request(signed_url, method='GET') + self.assertEqual(response.status, 200) + self.assertEqual(content, self.LOCAL_FILE) + + def test_create_signed_delete_url(self): + key = self.bucket.new_key('LogoToSign.jpg') + expiration = int(time.time() + 283473274) + signed_delete_url = key.generate_signed_url(expiration, + method='DELETE') + + response, content = HTTP.request(signed_delete_url, method='DELETE') + self.assertEqual(response.status, 204) + self.assertEqual(content, '') + + # Check that the key has actually been deleted. + self.assertRaises(storage.exceptions.NotFoundError, + key.reload_metadata) diff --git a/scripts/run_regression.sh b/scripts/run_regression.sh index 63a5318688de..038b9f22e446 100755 --- a/scripts/run_regression.sh +++ b/scripts/run_regression.sh @@ -36,3 +36,4 @@ fi # Run the regression tests for each tested package. python regression/run_regression.py --package datastore +python regression/run_regression.py --package storage