From 06232e96a4f5711b5bae8d5e5395f645fbc45ac0 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 10 Jun 2020 18:21:45 +0000 Subject: [PATCH] [storage] testing: use multiple projects We still need to use the old project for some tests. fixes #4033 fixes #4029 --- storage/cloud-client/acl_test.py | 8 ++ storage/cloud-client/hmac_samples_test.py | 10 ++- storage/cloud-client/iam_test.py | 37 ++++++-- storage/cloud-client/noxfile_config.py | 84 +++++++++++++++++++ storage/cloud-client/snippets_test.py | 41 +++++++-- .../uniform_bucket_level_access_test.py | 7 ++ 6 files changed, 172 insertions(+), 15 deletions(-) create mode 100644 storage/cloud-client/noxfile_config.py diff --git a/storage/cloud-client/acl_test.py b/storage/cloud-client/acl_test.py index 469d364ce300..eda2c7df18c6 100644 --- a/storage/cloud-client/acl_test.py +++ b/storage/cloud-client/acl_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import uuid from google.cloud import storage @@ -39,6 +40,11 @@ @pytest.fixture(scope="module") def test_bucket(): """Yields a bucket that is deleted after the test completes.""" + + # The new projects have uniform bucket-level access and our tests don't + # pass with those buckets. We need to use the old main project for now. + original_value = os.environ['GOOGLE_CLOUD_PROJECT'] + os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT'] bucket = None while bucket is None or bucket.exists(): bucket_name = "acl-test-{}".format(uuid.uuid4()) @@ -46,6 +52,8 @@ def test_bucket(): bucket.create() yield bucket bucket.delete(force=True) + # Set the value back. + os.environ['GOOGLE_CLOUD_PROJECT'] = original_value @pytest.fixture diff --git a/storage/cloud-client/hmac_samples_test.py b/storage/cloud-client/hmac_samples_test.py index f9fcb7509931..60eba240184e 100644 --- a/storage/cloud-client/hmac_samples_test.py +++ b/storage/cloud-client/hmac_samples_test.py @@ -31,8 +31,14 @@ import storage_get_hmac_key import storage_list_hmac_keys - -PROJECT_ID = os.environ["GOOGLE_CLOUD_PROJECT"] +# We are reaching maximum number of HMAC keys on the service account. +# We change the service account based on the value of +# RUN_TESTS_SESSION in noxfile_config.py. +# The reason we can not use multiple project is that our new projects +# are enforced to have +# 'constraints/iam.disableServiceAccountKeyCreation' policy. + +PROJECT_ID = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"] SERVICE_ACCOUNT_EMAIL = os.environ["HMAC_KEY_TEST_SERVICE_ACCOUNT"] STORAGE_CLIENT = storage.Client(project=PROJECT_ID) diff --git a/storage/cloud-client/iam_test.py b/storage/cloud-client/iam_test.py index 5186eb1905e4..d9277101ff34 100644 --- a/storage/cloud-client/iam_test.py +++ b/storage/cloud-client/iam_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re import time import uuid @@ -36,7 +37,7 @@ ) -@pytest.fixture +@pytest.fixture(scope="module") def bucket(): bucket = None while bucket is None or bucket.exists(): @@ -50,6 +51,26 @@ def bucket(): bucket.delete(force=True) +@pytest.fixture(scope="function") +def public_bucket(): + # The new projects don't allow to make a bucket available to public, so + # we need to use the old main project for now. + original_value = os.environ['GOOGLE_CLOUD_PROJECT'] + os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT'] + bucket = None + while bucket is None or bucket.exists(): + storage_client = storage.Client() + bucket_name = "test-iam-{}".format(uuid.uuid4()) + bucket = storage_client.bucket(bucket_name) + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + storage_client.create_bucket(bucket) + yield bucket + time.sleep(3) + bucket.delete(force=True) + # Set the value back. + os.environ['GOOGLE_CLOUD_PROJECT'] = original_value + + def test_view_bucket_iam_members(capsys, bucket): storage_view_bucket_iam_members.view_bucket_iam_members(bucket.name) assert re.match("Role: .*, Members: .*", capsys.readouterr().out) @@ -87,10 +108,12 @@ def test_add_bucket_conditional_iam_binding(bucket): ) -def test_remove_bucket_iam_member(bucket): - storage_remove_bucket_iam_member.remove_bucket_iam_member(bucket.name, ROLE, MEMBER) +def test_remove_bucket_iam_member(public_bucket): + storage_remove_bucket_iam_member.remove_bucket_iam_member( + public_bucket.name, ROLE, MEMBER) - policy = bucket.get_iam_policy(requested_policy_version=3) + policy = public_bucket.get_iam_policy(requested_policy_version=3) + print(policy) assert not any( binding["role"] == ROLE and MEMBER in binding["members"] for binding in policy.bindings @@ -114,13 +137,13 @@ def test_remove_bucket_conditional_iam_binding(bucket): ) -def test_set_bucket_public_iam(bucket): +def test_set_bucket_public_iam(public_bucket): role = "roles/storage.objectViewer" member = "allUsers" storage_set_bucket_public_iam.set_bucket_public_iam( - bucket.name, role, member + public_bucket.name, role, member ) - policy = bucket.get_iam_policy(requested_policy_version=3) + policy = public_bucket.get_iam_policy(requested_policy_version=3) assert any( binding["role"] == role and member in binding["members"] for binding in policy.bindings diff --git a/storage/cloud-client/noxfile_config.py b/storage/cloud-client/noxfile_config.py new file mode 100644 index 000000000000..edcea1530b63 --- /dev/null +++ b/storage/cloud-client/noxfile_config.py @@ -0,0 +1,84 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be inported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +import os + + +# We are reaching maximum number of HMAC keys on the service account. +# We change the service account based on the value of +# RUN_TESTS_SESSION. The reason we can not use multiple project is +# that our new projects are enforced to have +# 'constraints/iam.disableServiceAccountKeyCreation' policy. +def get_service_account_email(): + session = os.environ.get('RUN_TESTS_SESSION') + if session == 'py-3.6': + return ('py36-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + if session == 'py-3.7': + return ('py37-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + if session == 'py-3.8': + return ('py38-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + return os.environ['HMAC_KEY_TEST_SERVICE_ACCOUNT'] + + +# We change the value of CLOUD_KMS_KEY based on the value of +# RUN_TESTS_SESSION. +def get_cloud_kms_key(): + session = os.environ.get('RUN_TESTS_SESSION') + if session == 'py-3.6': + return ('projects/python-docs-samples-tests-py36/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.7': + return ('projects/python-docs-samples-tests-py37/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.8': + return ('projects/python-docs-samples-tests-py38/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + return os.environ['CLOUD_KMS_KEY'] + + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': { + 'HMAC_KEY_TEST_SERVICE_ACCOUNT': get_service_account_email(), + 'CLOUD_KMS_KEY': get_cloud_kms_key(), + # Some tests can not use multiple projects because of several reasons: + # 1. The new projects is enforced to have the + # 'constraints/iam.disableServiceAccountKeyCreation' policy. + # 2. The new projects buckets need to have universal permission model. + # For those tests, we'll use the original project. + 'MAIN_GOOGLE_CLOUD_PROJECT': 'python-docs-samples-tests' + }, +} diff --git a/storage/cloud-client/snippets_test.py b/storage/cloud-client/snippets_test.py index e6c3d1c1d967..edcf779391cb 100644 --- a/storage/cloud-client/snippets_test.py +++ b/storage/cloud-client/snippets_test.py @@ -105,6 +105,24 @@ def test_bucket(): bucket.delete(force=True) +@pytest.fixture(scope="function") +def test_public_bucket(): + # The new projects don't allow to make a bucket available to public, so + # for some tests we need to use the old main project for now. + original_value = os.environ['GOOGLE_CLOUD_PROJECT'] + os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT'] + bucket = None + while bucket is None or bucket.exists(): + storage_client = storage.Client() + bucket_name = "storage-snippets-test-{}".format(uuid.uuid4()) + bucket = storage_client.bucket(bucket_name) + storage_client.create_bucket(bucket) + yield bucket + bucket.delete(force=True) + # Set the value back. + os.environ['GOOGLE_CLOUD_PROJECT'] = original_value + + @pytest.fixture def test_blob(test_bucket): """Yields a blob that is deleted after the test completes.""" @@ -114,6 +132,15 @@ def test_blob(test_bucket): yield blob +@pytest.fixture(scope="function") +def test_public_blob(test_public_bucket): + """Yields a blob that is deleted after the test completes.""" + bucket = test_public_bucket + blob = bucket.blob("storage_snippets_test_sigil-{}".format(uuid.uuid4())) + blob.upload_from_string("Hello, is it me you're looking for?") + yield blob + + @pytest.fixture def test_bucket_create(): """Yields a bucket object that is deleted after the test completes.""" @@ -196,10 +223,11 @@ def test_delete_blob(test_blob): storage_delete_file.delete_blob(test_blob.bucket.name, test_blob.name) -def test_make_blob_public(test_blob): - storage_make_public.make_blob_public(test_blob.bucket.name, test_blob.name) +def test_make_blob_public(test_public_blob): + storage_make_public.make_blob_public( + test_public_blob.bucket.name, test_public_blob.name) - r = requests.get(test_blob.public_url) + r = requests.get(test_public_blob.public_url) assert r.text == "Hello, is it me you're looking for?" @@ -342,11 +370,12 @@ def test_get_service_account(capsys): assert "@gs-project-accounts.iam.gserviceaccount.com" in out -def test_download_public_file(test_blob): - storage_make_public.make_blob_public(test_blob.bucket.name, test_blob.name) +def test_download_public_file(test_public_blob): + storage_make_public.make_blob_public( + test_public_blob.bucket.name, test_public_blob.name) with tempfile.NamedTemporaryFile() as dest_file: storage_download_public_file.download_public_file( - test_blob.bucket.name, test_blob.name, dest_file.name + test_public_blob.bucket.name, test_public_blob.name, dest_file.name ) assert dest_file.read() == b"Hello, is it me you're looking for?" diff --git a/storage/cloud-client/uniform_bucket_level_access_test.py b/storage/cloud-client/uniform_bucket_level_access_test.py index 9d1e8e0b4b55..cf5a09f88279 100644 --- a/storage/cloud-client/uniform_bucket_level_access_test.py +++ b/storage/cloud-client/uniform_bucket_level_access_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import time from google.cloud import storage @@ -25,6 +26,10 @@ @pytest.fixture() def bucket(): """Yields a bucket that is deleted after the test completes.""" + # The new projects enforces uniform bucket level access, so + # we need to use the old main project for now. + original_value = os.environ['GOOGLE_CLOUD_PROJECT'] + os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT'] bucket = None while bucket is None or bucket.exists(): bucket_name = "uniform-bucket-level-access-{}".format(int(time.time())) @@ -33,6 +38,8 @@ def bucket(): yield bucket time.sleep(3) bucket.delete(force=True) + # Set the value back. + os.environ['GOOGLE_CLOUD_PROJECT'] = original_value def test_get_uniform_bucket_level_access(bucket, capsys):