Skip to content

Commit

Permalink
tests: refactor s3 compatibily tests naming relevance to s3 gate
Browse files Browse the repository at this point in the history 
S3 policy and cors tests were also moved to a more logical location
from the relevance to s3 gate point of view.

closes #707

Signed-off-by: Evgeniy Zayats <zayatsevgeniy@nspcc.io>
  • Loading branch information
Evgeniy Zayats committed Sep 26, 2024
1 parent 4a60a0a commit 0fe3c60
Show file tree
Hide file tree
Showing 13 changed files with 481 additions and 507 deletions.
20 changes: 10 additions & 10 deletions pytest_tests/lib/helpers/s3_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import allure
from dateutil.parser import parse
from s3 import s3_gate_bucket, s3_gate_object
from s3 import s3_bucket, s3_object

logger = logging.getLogger("NeoLogger")

Expand Down Expand Up @@ -77,7 +77,7 @@ def check_objects_in_bucket(
s3_client, bucket, expected_objects: list, unexpected_objects: Optional[list] = None
) -> None:
unexpected_objects = unexpected_objects or []
bucket_objects = s3_gate_object.list_objects_s3(s3_client, bucket)
bucket_objects = s3_object.list_objects_s3(s3_client, bucket)
assert len(bucket_objects) == len(expected_objects), f"Expected {len(expected_objects)} objects in the bucket"
for bucket_object in expected_objects:
assert bucket_object in bucket_objects, f"Expected object {bucket_object} in objects list {bucket_objects}"
Expand All @@ -92,17 +92,17 @@ def check_objects_in_bucket(
def try_to_get_objects_and_expect_error(s3_client, bucket: str, object_keys: list) -> None:
for obj in object_keys:
try:
s3_gate_object.get_object_s3(s3_client, bucket, obj)
s3_object.get_object_s3(s3_client, bucket, obj)
raise AssertionError(f"Object {obj} found in bucket {bucket}")
except Exception as err:
assert "The specified key does not exist" in str(err), f"Expected error in exception {err}"


@allure.step("Set versioning enable for bucket")
def set_bucket_versioning(s3_client, bucket: str, status: s3_gate_bucket.VersioningStatus):
s3_gate_bucket.get_bucket_versioning_status(s3_client, bucket)
s3_gate_bucket.set_bucket_versioning(s3_client, bucket, status=status)
bucket_status = s3_gate_bucket.get_bucket_versioning_status(s3_client, bucket)
def set_bucket_versioning(s3_client, bucket: str, status: s3_bucket.VersioningStatus):
s3_bucket.get_bucket_versioning_status(s3_client, bucket)
s3_bucket.set_bucket_versioning(s3_client, bucket, status=status)
bucket_status = s3_bucket.get_bucket_versioning_status(s3_client, bucket)
assert bucket_status == status.value, f"Expected {bucket_status} status. Got {status.value}"


Expand Down Expand Up @@ -132,13 +132,13 @@ def check_tags_by_object(
expected_tags: list,
unexpected_tags: Optional[list] = None,
) -> None:
actual_tags = s3_gate_object.get_object_tagging(s3_client, bucket, key_name)
actual_tags = s3_object.get_object_tagging(s3_client, bucket, key_name)
assert_tags(expected_tags=expected_tags, unexpected_tags=unexpected_tags, actual_tags=actual_tags)


@allure.step("Expected all tags are presented in bucket")
def check_tags_by_bucket(s3_client, bucket: str, expected_tags: list, unexpected_tags: Optional[list] = None) -> None:
actual_tags = s3_gate_bucket.get_bucket_tagging(s3_client, bucket)
actual_tags = s3_bucket.get_bucket_tagging(s3_client, bucket)
assert_tags(expected_tags=expected_tags, unexpected_tags=unexpected_tags, actual_tags=actual_tags)


Expand All @@ -151,7 +151,7 @@ def assert_object_lock_mode(
legal_hold_status: str = "OFF",
retain_period: Optional[int] = None,
):
object_dict = s3_gate_object.get_object_s3(s3_client, bucket, file_name, full_output=True)
object_dict = s3_object.get_object_s3(s3_client, bucket, file_name, full_output=True)
assert object_dict.get("ObjectLockMode") == object_lock_mode, f"Expected Object Lock Mode is {object_lock_mode}"
assert (
object_dict.get("ObjectLockLegalHoldStatus") == legal_hold_status
Expand Down
36 changes: 18 additions & 18 deletions pytest_tests/lib/s3/s3_gate_base.py → pytest_tests/lib/s3/s3_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from neofs_testlib.shell import Shell
from neofs_testlib.utils.wallet import get_last_public_key_from_wallet
from pytest import FixtureRequest
from s3 import s3_gate_bucket, s3_gate_object
from s3 import s3_bucket, s3_object

# Disable warnings on self-signed certificate which the
# boto library produces on requests to S3-gate in dev-env
Expand Down Expand Up @@ -51,7 +51,7 @@ def _run_with_passwd(cmd: str, password: str) -> str:
return cmd.decode()


class TestNeofsS3GateBase(NeofsEnvTestBase):
class TestNeofsS3Base(NeofsEnvTestBase):
s3_client: Any = None # noqa

@pytest.fixture(scope="class", autouse=True)
Expand Down Expand Up @@ -83,47 +83,47 @@ def s3_client( # noqa
client = configure_cli_client(access_key_id, secret_access_key, f"https://{neofs_env.s3_gw.address}")
else:
client = configure_boto3_client(access_key_id, secret_access_key, f"https://{neofs_env.s3_gw.address}")
TestNeofsS3GateBase.s3_client = client
TestNeofsS3GateBase.wallet = wallet
TestNeofsS3Base.s3_client = client
TestNeofsS3Base.wallet = wallet

@pytest.fixture
@allure.title("Create/delete bucket")
def bucket(self):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-1")
bucket = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-1")
yield bucket
self.delete_all_object_in_bucket(bucket)

@pytest.fixture
@allure.title("Create two buckets")
def two_buckets(self):
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-1")
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-1")
bucket_1 = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-1")
bucket_2 = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-1")
yield bucket_1, bucket_2
for bucket in [bucket_1, bucket_2]:
self.delete_all_object_in_bucket(bucket)

def delete_all_object_in_bucket(self, bucket):
versioning_status = s3_gate_bucket.get_bucket_versioning_status(self.s3_client, bucket)
if versioning_status == s3_gate_bucket.VersioningStatus.ENABLED.value:
versioning_status = s3_bucket.get_bucket_versioning_status(self.s3_client, bucket)
if versioning_status == s3_bucket.VersioningStatus.ENABLED.value:
# From versioned bucket we should delete all versions and delete markers of all objects
objects_versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket)
objects_versions = s3_object.list_objects_versions_s3(self.s3_client, bucket)
if objects_versions:
s3_gate_object.delete_object_versions_s3_without_dm(self.s3_client, bucket, objects_versions)
objects_delete_markers = s3_gate_object.list_objects_delete_markers_s3(self.s3_client, bucket)
s3_object.delete_object_versions_s3_without_dm(self.s3_client, bucket, objects_versions)
objects_delete_markers = s3_object.list_objects_delete_markers_s3(self.s3_client, bucket)
if objects_delete_markers:
s3_gate_object.delete_object_versions_s3_without_dm(self.s3_client, bucket, objects_delete_markers)
s3_object.delete_object_versions_s3_without_dm(self.s3_client, bucket, objects_delete_markers)

else:
# From non-versioned bucket it's sufficient to delete objects by key
objects = s3_gate_object.list_objects_s3(self.s3_client, bucket)
objects = s3_object.list_objects_s3(self.s3_client, bucket)
if objects:
s3_gate_object.delete_objects_s3(self.s3_client, bucket, objects)
objects_delete_markers = s3_gate_object.list_objects_delete_markers_s3(self.s3_client, bucket)
s3_object.delete_objects_s3(self.s3_client, bucket, objects)
objects_delete_markers = s3_object.list_objects_delete_markers_s3(self.s3_client, bucket)
if objects_delete_markers:
s3_gate_object.delete_object_versions_s3_without_dm(self.s3_client, bucket, objects_delete_markers)
s3_object.delete_object_versions_s3_without_dm(self.s3_client, bucket, objects_delete_markers)

# Delete the bucket itself
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket)
s3_bucket.delete_bucket_s3(self.s3_client, bucket)


@allure.step("Init S3 Credentials")
Expand Down
0 pytest_tests/lib/s3/s3_gate_bucket.py → pytest_tests/lib/s3/s3_bucket.py
View file
File renamed without changes.
2 changes: 1 addition & 1 deletion pytest_tests/lib/s3/s3_gate_object.py → pytest_tests/lib/s3/s3_object.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from helpers.aws_cli_client import AwsCliClient
from helpers.cli_helpers import log_command_execution
from helpers.common import ASSETS_DIR
from s3.s3_gate_bucket import S3_SYNC_WAIT_TIME
from s3.s3_bucket import S3_SYNC_WAIT_TIME

##########################################################
# Disabling warnings on self-signed certificate which the
Expand Down
118 changes: 25 additions & 93 deletions .../tests/services/s3_gate/test_s3_policy.py → ...s/s3/s3_extensions/test_s3_gate_policy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
set_bucket_versioning,
)
from helpers.utility import placement_policy_from_container
from s3 import s3_gate_bucket, s3_gate_object
from s3.s3_gate_base import TestNeofsS3GateBase
from s3 import s3_bucket, s3_object
from s3.s3_base import TestNeofsS3Base


def pytest_generate_tests(metafunc):
Expand All @@ -26,8 +26,7 @@ def pytest_generate_tests(metafunc):
)


@pytest.mark.s3_gate
class TestS3GatePolicy(TestNeofsS3GateBase):
class TestS3GatePolicy(TestNeofsS3Base):
def check_container_policy(self, bucket_name: str, expected_policy: str):
cid = search_container_by_name(self.wallet.path, bucket_name, shell=self.shell, endpoint=self.neofs_env.sn_rpc)
container_info: str = get_container(
Expand All @@ -50,30 +49,30 @@ def test_s3_bucket_location(self, simple_object_size):
file_name_2 = object_key_from_file_path(file_path_2)

with allure.step("Create two buckets with different bucket configuration"):
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="complex")
set_bucket_versioning(self.s3_client, bucket_1, s3_gate_bucket.VersioningStatus.ENABLED)
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-3")
set_bucket_versioning(self.s3_client, bucket_2, s3_gate_bucket.VersioningStatus.ENABLED)
list_buckets = s3_gate_bucket.list_buckets_s3(self.s3_client)
bucket_1 = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="complex")
set_bucket_versioning(self.s3_client, bucket_1, s3_bucket.VersioningStatus.ENABLED)
bucket_2 = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-3")
set_bucket_versioning(self.s3_client, bucket_2, s3_bucket.VersioningStatus.ENABLED)
list_buckets = s3_bucket.list_buckets_s3(self.s3_client)
assert (
bucket_1 in list_buckets and bucket_2 in list_buckets
), f"Expected two buckets {bucket_1, bucket_2}, got {list_buckets}"

# with allure.step("Check head buckets"):
head_1 = s3_gate_bucket.head_bucket(self.s3_client, bucket_1)
head_2 = s3_gate_bucket.head_bucket(self.s3_client, bucket_2)
head_1 = s3_bucket.head_bucket(self.s3_client, bucket_1)
head_2 = s3_bucket.head_bucket(self.s3_client, bucket_2)
assert head_1 == {} or head_1.get("HEAD") is None, "Expected head is empty"
assert head_2 == {} or head_2.get("HEAD") is None, "Expected head is empty"

with allure.step("Put objects into buckets"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_path_1)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket_2, file_path_2)
version_id_1 = s3_object.put_object_s3(self.s3_client, bucket_1, file_path_1)
version_id_2 = s3_object.put_object_s3(self.s3_client, bucket_2, file_path_2)
check_objects_in_bucket(self.s3_client, bucket_1, [file_name_1])
check_objects_in_bucket(self.s3_client, bucket_2, [file_name_2])

with allure.step("Check bucket location"):
bucket_loc_1 = s3_gate_bucket.get_bucket_location(self.s3_client, bucket_1)
bucket_loc_2 = s3_gate_bucket.get_bucket_location(self.s3_client, bucket_2)
bucket_loc_1 = s3_bucket.get_bucket_location(self.s3_client, bucket_1)
bucket_loc_2 = s3_bucket.get_bucket_location(self.s3_client, bucket_2)
assert bucket_loc_1 == "complex"
assert bucket_loc_2 == "rep-3"

Expand Down Expand Up @@ -111,29 +110,29 @@ def test_s3_bucket_location_from_config_file(self, simple_object_size):
file_name_2 = object_key_from_file_path(file_path_2)

with allure.step("Create two buckets with different bucket configuration"):
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="select")
set_bucket_versioning(self.s3_client, bucket_1, s3_gate_bucket.VersioningStatus.ENABLED)
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-2")
set_bucket_versioning(self.s3_client, bucket_2, s3_gate_bucket.VersioningStatus.ENABLED)
list_buckets = s3_gate_bucket.list_buckets_s3(self.s3_client)
bucket_1 = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="select")
set_bucket_versioning(self.s3_client, bucket_1, s3_bucket.VersioningStatus.ENABLED)
bucket_2 = s3_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-2")
set_bucket_versioning(self.s3_client, bucket_2, s3_bucket.VersioningStatus.ENABLED)
list_buckets = s3_bucket.list_buckets_s3(self.s3_client)
assert (
bucket_1 in list_buckets and bucket_2 in list_buckets
), f"Expected two buckets {bucket_1, bucket_2}, got {list_buckets}"

head_1 = s3_gate_bucket.head_bucket(self.s3_client, bucket_1)
head_2 = s3_gate_bucket.head_bucket(self.s3_client, bucket_2)
head_1 = s3_bucket.head_bucket(self.s3_client, bucket_1)
head_2 = s3_bucket.head_bucket(self.s3_client, bucket_2)
assert head_1 == {} or head_1.get("HEAD") is None, "Expected head is empty"
assert head_2 == {} or head_2.get("HEAD") is None, "Expected head is empty"

with allure.step("Put objects into buckets"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_path_1)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket_2, file_path_2)
version_id_1 = s3_object.put_object_s3(self.s3_client, bucket_1, file_path_1)
version_id_2 = s3_object.put_object_s3(self.s3_client, bucket_2, file_path_2)
check_objects_in_bucket(self.s3_client, bucket_1, [file_name_1])
check_objects_in_bucket(self.s3_client, bucket_2, [file_name_2])

with allure.step("Check bucket location"):
bucket_loc_1 = s3_gate_bucket.get_bucket_location(self.s3_client, bucket_1)
bucket_loc_2 = s3_gate_bucket.get_bucket_location(self.s3_client, bucket_2)
bucket_loc_1 = s3_bucket.get_bucket_location(self.s3_client, bucket_1)
bucket_loc_2 = s3_bucket.get_bucket_location(self.s3_client, bucket_2)
assert bucket_loc_1 == "select"
assert bucket_loc_2 == "rep-2"

Expand Down Expand Up @@ -164,70 +163,3 @@ def test_s3_bucket_location_from_config_file(self, simple_object_size):
nodes=self.neofs_env.storage_nodes,
)
assert copies_2 == 2

@allure.title("Test S3: bucket policy ")
def test_s3_bucket_policy(self):
with allure.step("Create bucket with default policy"):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)

with allure.step("GetBucketPolicy"):
s3_gate_bucket.get_bucket_policy(self.s3_client, bucket)

with allure.step("Put new policy"):
custom_policy = f"file://{os.getcwd()}/pytest_tests/data/bucket_policy.json"
custom_policy = {
"Version": "2008-10-17",
"Id": "aaaa-bbbb-cccc-dddd",
"Statement": [
{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": {"AWS": "*"},
"Action": ["s3:GetObject"],
"Resource": [f"arn:aws:s3:::{bucket}/*"],
}
],
}

s3_gate_bucket.put_bucket_policy(self.s3_client, bucket, custom_policy)
with allure.step("GetBucketPolicy"):
policy_1 = s3_gate_bucket.get_bucket_policy(self.s3_client, bucket)
print(policy_1)

@allure.title("Test S3: bucket policy ")
def test_s3_cors(self):
with allure.step("Create bucket without cors"):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)

with pytest.raises(Exception):
bucket_cors = s3_gate_bucket.get_bucket_cors(self.s3_client, bucket)

with allure.step("Put bucket cors"):
cors = {
"CORSRules": [
{
"AllowedOrigins": ["http://www.example.com"],
"AllowedHeaders": ["*"],
"AllowedMethods": ["PUT", "POST", "DELETE"],
"MaxAgeSeconds": 3000,
"ExposeHeaders": ["x-amz-server-side-encryption"],
},
{
"AllowedOrigins": ["*"],
"AllowedHeaders": ["Authorization"],
"AllowedMethods": ["GET"],
"MaxAgeSeconds": 3000,
},
]
}
s3_gate_bucket.put_bucket_cors(self.s3_client, bucket, cors)
bucket_cors = s3_gate_bucket.get_bucket_cors(self.s3_client, bucket)
assert bucket_cors == cors.get("CORSRules"), f"Expected corsrules must be {cors.get('CORSRules')}"

with allure.step("delete bucket cors"):
s3_gate_bucket.delete_bucket_cors(self.s3_client, bucket)

with pytest.raises(Exception):
bucket_cors = s3_gate_bucket.get_bucket_cors(self.s3_client, bucket)
Loading

0 comments on commit 0fe3c60

Please sign in to comment.