diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index ff6415d7a29..00000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[report] -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't complain if tests don't hit defensive assertion code: - raise NotImplementedError diff --git a/check_mypy.sh b/check_mypy.sh deleted file mode 100755 index f9c90e67d25..00000000000 --- a/check_mypy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -eux - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -rm -rf "${SCRIPT_DIR}/.collection_root" -mkdir -p "${SCRIPT_DIR}/.collection_root/ansible_collections/amazon/aws" -cp -r "${SCRIPT_DIR}/plugins" "${SCRIPT_DIR}/.collection_root/ansible_collections/amazon/aws/plugins" -cd "${SCRIPT_DIR}/.collection_root" -cp "${SCRIPT_DIR}/mypy.ini" . -mypy -p ansible_collections.amazon.aws.plugins \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 05eafedd49f..fa03b3c5c53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,18 @@ disable_error_code = ["import-untyped"] line-length = 120 [tool.ruff.lint] -# "F401" - unused-imports - We use these imports to maintaining historic Interfaces +# "F401" - unused-imports - We use these imports to maintain historic Interfaces # "E402" - import not at top of file - General Ansible style puts the documentation at the top. unfixable = ["F401"] ignore = ["F401", "E402"] + +[tool.pytest] +xfail_strict = true + +[tool.coverage.report] +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + # Don't complain if tests don't hit defensive assertion code: + "raise NotImplementedError", +] diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index a8fb1eab213..b20ec357dbe 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -9,3 +9,5 @@ virtualenv awscli # Used for comparing SSH Public keys to the Amazon fingerprints cryptography +# Used for recordings +placebo diff --git a/tests/integration/requirements.yml b/tests/integration/requirements.yml index c94dd39a7a8..c536f9f99a7 100644 --- a/tests/integration/requirements.yml +++ b/tests/integration/requirements.yml @@ -3,3 +3,4 @@ collections: - ansible.windows - ansible.utils # ipv6 filter - amazon.cloud # used by integration tests - rds_cluster_modify + - community.crypto # SSL certificate generation diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 28257b24788..7eb3f21a285 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- + # This file is part of Ansible # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -# pylint: disable=unused-import - -import pytest - -from .utils.amazon_placebo_fixtures import fixture_maybe_sleep -from .utils.amazon_placebo_fixtures import fixture_placeboify +# While it may seem appropriate to import our custom fixtures here, the pytest_ansible pytest plugin +# isn't as agressive as the ansible_test._util.target.pytest.plugins.ansible_pytest_collections plugin +# when it comes to rewriting the import paths and as such we can't import fixtures via their +# absolute import path or across collections. diff --git a/tests/unit/plugins/conftest.py b/tests/unit/plugins/conftest.py new file mode 100644 index 00000000000..e9e86fc03f4 --- /dev/null +++ b/tests/unit/plugins/conftest.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# pylint: disable=unused-import + +import pytest + +from ansible_collections.amazon.aws.tests.unit.utils.amazon_placebo_fixtures import fixture_maybe_sleep +from ansible_collections.amazon.aws.tests.unit.utils.amazon_placebo_fixtures import fixture_placeboify diff --git a/tests/unit/module_utils/__init__.py b/tests/unit/plugins/module_utils/__init__.py similarity index 100% rename from tests/unit/module_utils/__init__.py rename to tests/unit/plugins/module_utils/__init__.py diff --git a/tests/unit/module_utils/arn/__init__.py b/tests/unit/plugins/module_utils/arn/__init__.py similarity index 100% rename from tests/unit/module_utils/arn/__init__.py rename to tests/unit/plugins/module_utils/arn/__init__.py diff --git a/tests/unit/module_utils/arn/test_is_outpost_arn.py b/tests/unit/plugins/module_utils/arn/test_is_outpost_arn.py similarity index 100% rename from tests/unit/module_utils/arn/test_is_outpost_arn.py rename to tests/unit/plugins/module_utils/arn/test_is_outpost_arn.py diff --git a/tests/unit/module_utils/arn/test_parse_aws_arn.py b/tests/unit/plugins/module_utils/arn/test_parse_aws_arn.py similarity index 100% rename from tests/unit/module_utils/arn/test_parse_aws_arn.py rename to tests/unit/plugins/module_utils/arn/test_parse_aws_arn.py diff --git a/tests/unit/module_utils/arn/test_validate_aws_arn.py b/tests/unit/plugins/module_utils/arn/test_validate_aws_arn.py similarity index 100% rename from tests/unit/module_utils/arn/test_validate_aws_arn.py rename to tests/unit/plugins/module_utils/arn/test_validate_aws_arn.py diff --git a/tests/unit/module_utils/autoscaling/test_autoscaling_error_handler.py b/tests/unit/plugins/module_utils/autoscaling/test_autoscaling_error_handler.py similarity index 100% rename from tests/unit/module_utils/autoscaling/test_autoscaling_error_handler.py rename to tests/unit/plugins/module_utils/autoscaling/test_autoscaling_error_handler.py diff --git a/tests/unit/module_utils/autoscaling/test_autoscaling_resource_transforms.py b/tests/unit/plugins/module_utils/autoscaling/test_autoscaling_resource_transforms.py similarity index 100% rename from tests/unit/module_utils/autoscaling/test_autoscaling_resource_transforms.py rename to tests/unit/plugins/module_utils/autoscaling/test_autoscaling_resource_transforms.py diff --git a/tests/unit/module_utils/botocore/__init__.py b/tests/unit/plugins/module_utils/botocore/__init__.py similarity index 100% rename from tests/unit/module_utils/botocore/__init__.py rename to tests/unit/plugins/module_utils/botocore/__init__.py diff --git a/tests/unit/module_utils/botocore/test_aws_region.py b/tests/unit/plugins/module_utils/botocore/test_aws_region.py similarity index 100% rename from tests/unit/module_utils/botocore/test_aws_region.py rename to tests/unit/plugins/module_utils/botocore/test_aws_region.py diff --git a/tests/unit/module_utils/botocore/test_boto3_conn.py b/tests/unit/plugins/module_utils/botocore/test_boto3_conn.py similarity index 100% rename from tests/unit/module_utils/botocore/test_boto3_conn.py rename to tests/unit/plugins/module_utils/botocore/test_boto3_conn.py diff --git a/tests/unit/module_utils/botocore/test_connection_info.py b/tests/unit/plugins/module_utils/botocore/test_connection_info.py similarity index 100% rename from tests/unit/module_utils/botocore/test_connection_info.py rename to tests/unit/plugins/module_utils/botocore/test_connection_info.py diff --git a/tests/unit/module_utils/botocore/test_is_boto3_error_code.py b/tests/unit/plugins/module_utils/botocore/test_is_boto3_error_code.py similarity index 100% rename from tests/unit/module_utils/botocore/test_is_boto3_error_code.py rename to tests/unit/plugins/module_utils/botocore/test_is_boto3_error_code.py diff --git a/tests/unit/module_utils/botocore/test_is_boto3_error_message.py b/tests/unit/plugins/module_utils/botocore/test_is_boto3_error_message.py similarity index 100% rename from tests/unit/module_utils/botocore/test_is_boto3_error_message.py rename to tests/unit/plugins/module_utils/botocore/test_is_boto3_error_message.py diff --git a/tests/unit/module_utils/botocore/test_merge_botocore_config.py b/tests/unit/plugins/module_utils/botocore/test_merge_botocore_config.py similarity index 100% rename from tests/unit/module_utils/botocore/test_merge_botocore_config.py rename to tests/unit/plugins/module_utils/botocore/test_merge_botocore_config.py diff --git a/tests/unit/module_utils/botocore/test_normalize_boto3_result.py b/tests/unit/plugins/module_utils/botocore/test_normalize_boto3_result.py similarity index 100% rename from tests/unit/module_utils/botocore/test_normalize_boto3_result.py rename to tests/unit/plugins/module_utils/botocore/test_normalize_boto3_result.py diff --git a/tests/unit/module_utils/botocore/test_sdk_versions.py b/tests/unit/plugins/module_utils/botocore/test_sdk_versions.py similarity index 100% rename from tests/unit/module_utils/botocore/test_sdk_versions.py rename to tests/unit/plugins/module_utils/botocore/test_sdk_versions.py diff --git a/tests/unit/module_utils/cloud/__init__.py b/tests/unit/plugins/module_utils/cloud/__init__.py similarity index 100% rename from tests/unit/module_utils/cloud/__init__.py rename to tests/unit/plugins/module_utils/cloud/__init__.py diff --git a/tests/unit/module_utils/cloud/test_backoff_iterator.py b/tests/unit/plugins/module_utils/cloud/test_backoff_iterator.py similarity index 100% rename from tests/unit/module_utils/cloud/test_backoff_iterator.py rename to tests/unit/plugins/module_utils/cloud/test_backoff_iterator.py diff --git a/tests/unit/module_utils/cloud/test_cloud_retry.py b/tests/unit/plugins/module_utils/cloud/test_cloud_retry.py similarity index 100% rename from tests/unit/module_utils/cloud/test_cloud_retry.py rename to tests/unit/plugins/module_utils/cloud/test_cloud_retry.py diff --git a/tests/unit/module_utils/cloud/test_decorator_generation.py b/tests/unit/plugins/module_utils/cloud/test_decorator_generation.py similarity index 100% rename from tests/unit/module_utils/cloud/test_decorator_generation.py rename to tests/unit/plugins/module_utils/cloud/test_decorator_generation.py diff --git a/tests/unit/module_utils/cloud/test_retries_found.py b/tests/unit/plugins/module_utils/cloud/test_retries_found.py similarity index 100% rename from tests/unit/module_utils/cloud/test_retries_found.py rename to tests/unit/plugins/module_utils/cloud/test_retries_found.py diff --git a/tests/unit/module_utils/cloud/test_retry_func.py b/tests/unit/plugins/module_utils/cloud/test_retry_func.py similarity index 100% rename from tests/unit/module_utils/cloud/test_retry_func.py rename to tests/unit/plugins/module_utils/cloud/test_retry_func.py diff --git a/tests/unit/module_utils/conftest.py b/tests/unit/plugins/module_utils/conftest.py similarity index 99% rename from tests/unit/module_utils/conftest.py rename to tests/unit/plugins/module_utils/conftest.py index 4a79faf8e75..e670a579a28 100644 --- a/tests/unit/module_utils/conftest.py +++ b/tests/unit/plugins/module_utils/conftest.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright (c) 2017 Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) diff --git a/tests/unit/module_utils/ec2/test_determine_iam_role.py b/tests/unit/plugins/module_utils/ec2/test_determine_iam_role.py similarity index 100% rename from tests/unit/module_utils/ec2/test_determine_iam_role.py rename to tests/unit/plugins/module_utils/ec2/test_determine_iam_role.py diff --git a/tests/unit/module_utils/elbv2/__init__.py b/tests/unit/plugins/module_utils/elbv2/__init__.py similarity index 100% rename from tests/unit/module_utils/elbv2/__init__.py rename to tests/unit/plugins/module_utils/elbv2/__init__.py diff --git a/tests/unit/module_utils/elbv2/test_elbv2.py b/tests/unit/plugins/module_utils/elbv2/test_elbv2.py similarity index 100% rename from tests/unit/module_utils/elbv2/test_elbv2.py rename to tests/unit/plugins/module_utils/elbv2/test_elbv2.py diff --git a/tests/unit/module_utils/elbv2/test_listener_rules.py b/tests/unit/plugins/module_utils/elbv2/test_listener_rules.py similarity index 100% rename from tests/unit/module_utils/elbv2/test_listener_rules.py rename to tests/unit/plugins/module_utils/elbv2/test_listener_rules.py diff --git a/tests/unit/module_utils/elbv2/test_prune.py b/tests/unit/plugins/module_utils/elbv2/test_prune.py similarity index 100% rename from tests/unit/module_utils/elbv2/test_prune.py rename to tests/unit/plugins/module_utils/elbv2/test_prune.py diff --git a/tests/unit/module_utils/errors/aws_error_handler/test_common_handler.py b/tests/unit/plugins/module_utils/errors/aws_error_handler/test_common_handler.py similarity index 100% rename from tests/unit/module_utils/errors/aws_error_handler/test_common_handler.py rename to tests/unit/plugins/module_utils/errors/aws_error_handler/test_common_handler.py diff --git a/tests/unit/module_utils/errors/aws_error_handler/test_deletion_handler.py b/tests/unit/plugins/module_utils/errors/aws_error_handler/test_deletion_handler.py similarity index 100% rename from tests/unit/module_utils/errors/aws_error_handler/test_deletion_handler.py rename to tests/unit/plugins/module_utils/errors/aws_error_handler/test_deletion_handler.py diff --git a/tests/unit/module_utils/errors/aws_error_handler/test_list_handler.py b/tests/unit/plugins/module_utils/errors/aws_error_handler/test_list_handler.py similarity index 100% rename from tests/unit/module_utils/errors/aws_error_handler/test_list_handler.py rename to tests/unit/plugins/module_utils/errors/aws_error_handler/test_list_handler.py diff --git a/tests/unit/module_utils/exceptions/__init__.py b/tests/unit/plugins/module_utils/exceptions/__init__.py similarity index 100% rename from tests/unit/module_utils/exceptions/__init__.py rename to tests/unit/plugins/module_utils/exceptions/__init__.py diff --git a/tests/unit/module_utils/exceptions/test_exceptions.py b/tests/unit/plugins/module_utils/exceptions/test_exceptions.py similarity index 100% rename from tests/unit/module_utils/exceptions/test_exceptions.py rename to tests/unit/plugins/module_utils/exceptions/test_exceptions.py diff --git a/tests/unit/module_utils/iam/test_iam_error_handler.py b/tests/unit/plugins/module_utils/iam/test_iam_error_handler.py similarity index 100% rename from tests/unit/module_utils/iam/test_iam_error_handler.py rename to tests/unit/plugins/module_utils/iam/test_iam_error_handler.py diff --git a/tests/unit/module_utils/iam/test_iam_resource_transforms.py b/tests/unit/plugins/module_utils/iam/test_iam_resource_transforms.py similarity index 100% rename from tests/unit/module_utils/iam/test_iam_resource_transforms.py rename to tests/unit/plugins/module_utils/iam/test_iam_resource_transforms.py diff --git a/tests/unit/module_utils/iam/test_validate_iam_identifiers.py b/tests/unit/plugins/module_utils/iam/test_validate_iam_identifiers.py similarity index 100% rename from tests/unit/module_utils/iam/test_validate_iam_identifiers.py rename to tests/unit/plugins/module_utils/iam/test_validate_iam_identifiers.py diff --git a/tests/unit/module_utils/modules/__init__.py b/tests/unit/plugins/module_utils/modules/__init__.py similarity index 100% rename from tests/unit/module_utils/modules/__init__.py rename to tests/unit/plugins/module_utils/modules/__init__.py diff --git a/tests/unit/module_utils/modules/ansible_aws_module/__init__.py b/tests/unit/plugins/module_utils/modules/ansible_aws_module/__init__.py similarity index 100% rename from tests/unit/module_utils/modules/ansible_aws_module/__init__.py rename to tests/unit/plugins/module_utils/modules/ansible_aws_module/__init__.py diff --git a/tests/unit/module_utils/modules/ansible_aws_module/test_fail_json_aws.py b/tests/unit/plugins/module_utils/modules/ansible_aws_module/test_fail_json_aws.py similarity index 100% rename from tests/unit/module_utils/modules/ansible_aws_module/test_fail_json_aws.py rename to tests/unit/plugins/module_utils/modules/ansible_aws_module/test_fail_json_aws.py diff --git a/tests/unit/module_utils/modules/ansible_aws_module/test_minimal_versions.py b/tests/unit/plugins/module_utils/modules/ansible_aws_module/test_minimal_versions.py similarity index 100% rename from tests/unit/module_utils/modules/ansible_aws_module/test_minimal_versions.py rename to tests/unit/plugins/module_utils/modules/ansible_aws_module/test_minimal_versions.py diff --git a/tests/unit/module_utils/modules/ansible_aws_module/test_passthrough.py b/tests/unit/plugins/module_utils/modules/ansible_aws_module/test_passthrough.py similarity index 100% rename from tests/unit/module_utils/modules/ansible_aws_module/test_passthrough.py rename to tests/unit/plugins/module_utils/modules/ansible_aws_module/test_passthrough.py diff --git a/tests/unit/module_utils/modules/ansible_aws_module/test_require_at_least.py b/tests/unit/plugins/module_utils/modules/ansible_aws_module/test_require_at_least.py similarity index 100% rename from tests/unit/module_utils/modules/ansible_aws_module/test_require_at_least.py rename to tests/unit/plugins/module_utils/modules/ansible_aws_module/test_require_at_least.py diff --git a/tests/unit/module_utils/policy/__init__.py b/tests/unit/plugins/module_utils/policy/__init__.py similarity index 100% rename from tests/unit/module_utils/policy/__init__.py rename to tests/unit/plugins/module_utils/policy/__init__.py diff --git a/tests/unit/module_utils/policy/test_canonicalize.py b/tests/unit/plugins/module_utils/policy/test_canonicalize.py similarity index 100% rename from tests/unit/module_utils/policy/test_canonicalize.py rename to tests/unit/plugins/module_utils/policy/test_canonicalize.py diff --git a/tests/unit/module_utils/policy/test_compare_policies.py b/tests/unit/plugins/module_utils/policy/test_compare_policies.py similarity index 100% rename from tests/unit/module_utils/policy/test_compare_policies.py rename to tests/unit/plugins/module_utils/policy/test_compare_policies.py diff --git a/tests/unit/module_utils/policy/test_py3cmp.py b/tests/unit/plugins/module_utils/policy/test_py3cmp.py similarity index 100% rename from tests/unit/module_utils/policy/test_py3cmp.py rename to tests/unit/plugins/module_utils/policy/test_py3cmp.py diff --git a/tests/unit/module_utils/policy/test_simple_hashable_policy.py b/tests/unit/plugins/module_utils/policy/test_simple_hashable_policy.py similarity index 100% rename from tests/unit/module_utils/policy/test_simple_hashable_policy.py rename to tests/unit/plugins/module_utils/policy/test_simple_hashable_policy.py diff --git a/tests/unit/module_utils/retries/__init__.py b/tests/unit/plugins/module_utils/retries/__init__.py similarity index 100% rename from tests/unit/module_utils/retries/__init__.py rename to tests/unit/plugins/module_utils/retries/__init__.py diff --git a/tests/unit/module_utils/retries/test_awsretry.py b/tests/unit/plugins/module_utils/retries/test_awsretry.py similarity index 100% rename from tests/unit/module_utils/retries/test_awsretry.py rename to tests/unit/plugins/module_utils/retries/test_awsretry.py diff --git a/tests/unit/module_utils/retries/test_botocore_exception_maybe.py b/tests/unit/plugins/module_utils/retries/test_botocore_exception_maybe.py similarity index 100% rename from tests/unit/module_utils/retries/test_botocore_exception_maybe.py rename to tests/unit/plugins/module_utils/retries/test_botocore_exception_maybe.py diff --git a/tests/unit/module_utils/retries/test_retry_wrapper.py b/tests/unit/plugins/module_utils/retries/test_retry_wrapper.py similarity index 100% rename from tests/unit/module_utils/retries/test_retry_wrapper.py rename to tests/unit/plugins/module_utils/retries/test_retry_wrapper.py diff --git a/tests/unit/plugins/module_utils/s3/test_endpoints.py b/tests/unit/plugins/module_utils/s3/test_endpoints.py new file mode 100644 index 00000000000..98d46958658 --- /dev/null +++ b/tests/unit/plugins/module_utils/s3/test_endpoints.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# (c) 2021 Red Hat Inc. +# +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from unittest.mock import patch + +import pytest + +from ansible_collections.amazon.aws.plugins.module_utils import s3 + +mod_urlparse = "ansible_collections.amazon.aws.plugins.module_utils.s3.urlparse" + + +class UrlInfo: + def __init__(self, scheme=None, hostname=None, port=None): + self.hostname = hostname + self.scheme = scheme + self.port = port + + +@patch(mod_urlparse) +def test_is_fakes3_with_none_arg(m_urlparse): + m_urlparse.side_effect = SystemExit(1) + result = s3.is_fakes3(None) + assert not result + m_urlparse.assert_not_called() + + +@pytest.mark.parametrize( + "url,scheme,result", + [ + ("https://test-s3.amazon.com", "https", False), + ("fakes3://test-s3.amazon.com", "fakes3", True), + ("fakes3s://test-s3.amazon.com", "fakes3s", True), + ], +) +@patch(mod_urlparse) +def test_is_fakes3(m_urlparse, url, scheme, result): + m_urlparse.return_value = UrlInfo(scheme=scheme) + assert result == s3.is_fakes3(url) + m_urlparse.assert_called_with(url) + + +@pytest.mark.parametrize( + "url,urlinfo,endpoint", + [ + ( + "fakes3://test-s3.amazon.com", + {"scheme": "fakes3", "hostname": "test-s3.amazon.com"}, + {"endpoint": "http://test-s3.amazon.com:80", "use_ssl": False}, + ), + ( + "fakes3://test-s3.amazon.com:8080", + {"scheme": "fakes3", "hostname": "test-s3.amazon.com", "port": 8080}, + {"endpoint": "http://test-s3.amazon.com:8080", "use_ssl": False}, + ), + ( + "fakes3s://test-s3.amazon.com", + {"scheme": "fakes3s", "hostname": "test-s3.amazon.com"}, + {"endpoint": "https://test-s3.amazon.com:443", "use_ssl": True}, + ), + ( + "fakes3s://test-s3.amazon.com:9096", + {"scheme": "fakes3s", "hostname": "test-s3.amazon.com", "port": 9096}, + {"endpoint": "https://test-s3.amazon.com:9096", "use_ssl": True}, + ), + ], +) +@patch(mod_urlparse) +def test_parse_fakes3_endpoint(m_urlparse, url, urlinfo, endpoint): + m_urlparse.return_value = UrlInfo(**urlinfo) + result = s3.parse_fakes3_endpoint(url) + assert endpoint == result + m_urlparse.assert_called_with(url) + + +@pytest.mark.parametrize( + "url,scheme,use_ssl", + [ + ("https://test-s3-ceph.amazon.com", "https", True), + ("http://test-s3-ceph.amazon.com", "http", False), + ], +) +@patch(mod_urlparse) +def test_parse_ceph_endpoint(m_urlparse, url, scheme, use_ssl): + m_urlparse.return_value = UrlInfo(scheme=scheme) + result = s3.parse_ceph_endpoint(url) + assert result == {"endpoint": url, "use_ssl": use_ssl} + m_urlparse.assert_called_with(url) diff --git a/tests/unit/module_utils/test_s3.py b/tests/unit/plugins/module_utils/s3/test_etags.py similarity index 100% rename from tests/unit/module_utils/test_s3.py rename to tests/unit/plugins/module_utils/s3/test_etags.py diff --git a/tests/unit/plugins/module_utils/s3/test_s3_error_handler.py b/tests/unit/plugins/module_utils/s3/test_s3_error_handler.py new file mode 100644 index 00000000000..d1dcc5b6755 --- /dev/null +++ b/tests/unit/plugins/module_utils/s3/test_s3_error_handler.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +try: + import botocore +except ImportError: + pass + +import pytest + +from ansible_collections.amazon.aws.plugins.module_utils.botocore import HAS_BOTO3 +from ansible_collections.amazon.aws.plugins.module_utils.s3 import AnsibleS3Error +from ansible_collections.amazon.aws.plugins.module_utils.s3 import AnsibleS3PermissionsError +from ansible_collections.amazon.aws.plugins.module_utils.s3 import AnsibleS3Sigv4RequiredError +from ansible_collections.amazon.aws.plugins.module_utils.s3 import AnsibleS3SupportError +from ansible_collections.amazon.aws.plugins.module_utils.s3 import S3ErrorHandler + +if not HAS_BOTO3: + pytestmark = pytest.mark.skip("test_s3_error_handler.py requires the python modules 'boto3' and 'botocore'") + + +class TestS3DeletionHandler: + def test_no_failures(self): + self.counter = 0 + + @S3ErrorHandler.deletion_error_handler("no error") + def no_failures(): + self.counter += 1 + + no_failures() + assert self.counter == 1 + + def test_client_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "MalformedPolicyDocument", + "Message": "Policy document should not specify a principal", + } + } + + @S3ErrorHandler.deletion_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3Error) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) + + def test_ignore_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "404", + "Message": "Not found", + } + } + + @S3ErrorHandler.deletion_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "I couldn't find it") + + ret_val = raise_client_error() + assert self.counter == 1 + assert ret_val is False + + +class TestS3ListHandler: + def test_no_failures(self): + self.counter = 0 + + @S3ErrorHandler.list_error_handler("no error") + def no_failures(): + self.counter += 1 + + no_failures() + assert self.counter == 1 + + def test_client_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "MalformedPolicyDocument", + "Message": "Policy document should not specify a principal.", + } + } + + @S3ErrorHandler.list_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3Error) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) + + def test_list_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "404", + "Message": "Not found", + } + } + + @S3ErrorHandler.list_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "I couldn't find it") + + ret_val = raise_client_error() + assert self.counter == 1 + assert ret_val is None + + +class TestS3CommonHandler: + def test_no_failures(self): + self.counter = 0 + + @S3ErrorHandler.common_error_handler("no error") + def no_failures(): + self.counter += 1 + + no_failures() + assert self.counter == 1 + + def test_client_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "MalformedPolicyDocument", + "Message": "Policy document should not specify a principal.", + } + } + + @S3ErrorHandler.common_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3Error) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) + + def test_permissions_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "AccessDenied", + "Message": "Forbidden", + } + } + + @S3ErrorHandler.common_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3PermissionsError) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) + + def test_not_implemented_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "XNotImplemented", + "Message": "The request you provided implies functionality that is not implemented.", + } + } + + @S3ErrorHandler.common_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3SupportError) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) + + def test_endpoint_error(self): + self.counter = 0 + + @S3ErrorHandler.common_error_handler("do something") + def raise_connection_error(): + self.counter += 1 + raise botocore.exceptions.EndpointConnectionError(endpoint_url="junk.endpoint") + + with pytest.raises(AnsibleS3Error) as e_info: + raise_connection_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.BotoCoreError) + assert "do something" in raised.message + assert "junk.endpoint" in str(raised.exception) + + def test_sigv4_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "InvalidArgument", + "Message": "Requests specifying Server Side Encryption with AWS KMS managed keys require AWS Signature Version 4", + } + } + + @S3ErrorHandler.common_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3Sigv4RequiredError) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) + + def test_boto3_error(self): + self.counter = 0 + err_response = { + "Error": { + "Code": "MalformedPolicyDocument", + "Message": "Policy document should not specify a principal.", + } + } + + @S3ErrorHandler.common_error_handler("do something") + def raise_client_error(): + self.counter += 1 + raise botocore.exceptions.ClientError(err_response, "Something bad") + + with pytest.raises(AnsibleS3Error) as e_info: + raise_client_error() + assert self.counter == 1 + raised = e_info.value + assert isinstance(raised.exception, botocore.exceptions.ClientError) + assert "do something" in raised.message + assert "Something bad" in str(raised.exception) diff --git a/tests/unit/plugins/module_utils/s3/test_validate_bucket_name.py b/tests/unit/plugins/module_utils/s3/test_validate_bucket_name.py new file mode 100644 index 00000000000..45d935752d0 --- /dev/null +++ b/tests/unit/plugins/module_utils/s3/test_validate_bucket_name.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# +# (c) 2021 Red Hat Inc. +# +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +import pytest + +from ansible_collections.amazon.aws.plugins.module_utils import s3 + + +@pytest.mark.parametrize( + "bucket_name,result", + [ + ("docexamplebucket1", None), + ("log-delivery-march-2020", None), + ("my-hosted-content", None), + ("docexamplewebsite.com", None), + ("www.docexamplewebsite.com", None), + ("my.example.s3.bucket", None), + ("doc", None), + ("doc_example_bucket", "invalid character(s) found in the bucket name"), + ("DocExampleBucket", "invalid character(s) found in the bucket name"), + ("doc-example-bucket-", "bucket names must begin and end with a letter or number"), + ( + "this.string.has.more.than.63.characters.so.it.should.not.passed.the.validated", + "the length of an S3 bucket cannot exceed 63 characters", + ), + ("my", "the length of an S3 bucket must be at least 3 characters"), + ], +) +def test_validate_bucket_name(bucket_name, result): + assert result == s3.validate_bucket_name(bucket_name) diff --git a/tests/unit/module_utils/test_acm.py b/tests/unit/plugins/module_utils/test_acm.py similarity index 100% rename from tests/unit/module_utils/test_acm.py rename to tests/unit/plugins/module_utils/test_acm.py diff --git a/tests/unit/module_utils/test_cloudfront_facts.py b/tests/unit/plugins/module_utils/test_cloudfront_facts.py similarity index 100% rename from tests/unit/module_utils/test_cloudfront_facts.py rename to tests/unit/plugins/module_utils/test_cloudfront_facts.py diff --git a/tests/unit/module_utils/test_get_aws_account_id.py b/tests/unit/plugins/module_utils/test_get_aws_account_id.py similarity index 100% rename from tests/unit/module_utils/test_get_aws_account_id.py rename to tests/unit/plugins/module_utils/test_get_aws_account_id.py diff --git a/tests/unit/module_utils/test_rds.py b/tests/unit/plugins/module_utils/test_rds.py similarity index 100% rename from tests/unit/module_utils/test_rds.py rename to tests/unit/plugins/module_utils/test_rds.py diff --git a/tests/unit/plugins/module_utils/test_s3.py b/tests/unit/plugins/module_utils/test_s3.py new file mode 100644 index 00000000000..3770064c5b8 --- /dev/null +++ b/tests/unit/plugins/module_utils/test_s3.py @@ -0,0 +1,295 @@ +# +# (c) 2021 Red Hat Inc. +# +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +import random +import string +from unittest.mock import MagicMock +from unittest.mock import call +from unittest.mock import patch + +import pytest + +from ansible_collections.amazon.aws.plugins.module_utils import s3 + +try: + import botocore +except ImportError: + pass + + +def generate_random_string(size, include_digits=True): + buffer = string.ascii_lowercase + if include_digits: + buffer += string.digits + + return "".join(random.choice(buffer) for i in range(size)) + + +@pytest.mark.parametrize("parts", range(0, 10, 3)) +@pytest.mark.parametrize("version", [True, False]) +def test_s3_head_objects(parts, version): + client = MagicMock() + + s3bucket_name = f"s3-bucket-{generate_random_string(8, False)}" + s3bucket_object = f"s3-bucket-object-{generate_random_string(8, False)}" + versionId = None + if version: + versionId = random.randint(0, 1000) + + total = 0 + for head in s3.s3_head_objects(client, parts, s3bucket_name, s3bucket_object, versionId): + assert head == client.head_object.return_value + total += 1 + + assert total == parts + params = {"Bucket": s3bucket_name, "Key": s3bucket_object} + if versionId: + params["VersionId"] = versionId + + api_calls = [call(PartNumber=i, **params) for i in range(1, parts + 1)] + client.head_object.assert_has_calls(api_calls, any_order=True) + + +def raise_botoclient_exception(): + params = { + "Error": {"Code": 1, "Message": "Something went wrong"}, + "ResponseMetadata": {"RequestId": "01234567-89ab-cdef-0123-456789abcdef"}, + } + return botocore.exceptions.ClientError(params, "some_called_method") + + +@pytest.mark.parametrize("use_file", [False, True]) +@pytest.mark.parametrize("parts", range(0, 10, 3)) +@patch("ansible_collections.amazon.aws.plugins.module_utils.s3.md5") +@patch("ansible_collections.amazon.aws.plugins.module_utils.s3.s3_head_objects") +def test_calculate_checksum(m_s3_head_objects, m_s3_md5, use_file, parts, tmp_path): + client = MagicMock() + mock_md5 = m_s3_md5.return_value + + mock_md5.digest.return_value = b"1" + mock_md5.hexdigest.return_value = "".join(["f" for i in range(32)]) + + m_s3_head_objects.return_value = [{"ContentLength": f"{int(i + 1)}"} for i in range(parts)] + + content = b'"f20e84ac3d0c33cea77b3f29e3323a09"' + test_function = s3.calculate_checksum_with_content + if use_file: + test_function = s3.calculate_checksum_with_file + test_dir = tmp_path / "test_s3" + test_dir.mkdir() + etag_file = test_dir / "etag.bin" + etag_file.write_bytes(content) + + content = str(etag_file) + + s3bucket_name = f"s3-bucket-{generate_random_string(8, False)}" + s3bucket_object = f"s3-bucket-object-{generate_random_string(8, False)}" + version = random.randint(0, 1000) + + result = test_function(client, parts, s3bucket_name, s3bucket_object, version, content) + + expected = f'"{mock_md5.hexdigest.return_value}-{parts}"' + assert result == expected + + mock_md5.digest.assert_has_calls([call() for i in range(parts)]) + mock_md5.hexdigest.assert_called_once() + + m_s3_head_objects.assert_called_once_with(client, parts, s3bucket_name, s3bucket_object, version) + + +@pytest.mark.parametrize("etag_multipart", [True, False]) +@patch("ansible_collections.amazon.aws.plugins.module_utils.s3.calculate_checksum_with_file") +def test_calculate_etag(m_checksum_file, etag_multipart): + module = MagicMock() + client = MagicMock() + + module.fail_json_aws.side_effect = SystemExit(2) + module.md5.return_value = generate_random_string(32) + + s3bucket_name = f"s3-bucket-{generate_random_string(8, False)}" + s3bucket_object = f"s3-bucket-object-{generate_random_string(8, False)}" + version = random.randint(0, 1000) + parts = 3 + + etag = '"f20e84ac3d0c33cea77b3f29e3323a09"' + digest = '"9aa254f7f76fd14435b21e9448525b99"' + + file_name = generate_random_string(32) + + if not etag_multipart: + result = s3.calculate_etag(module, file_name, etag, client, s3bucket_name, s3bucket_object, version) + assert result == f'"{module.md5.return_value}"' + module.md5.assert_called_once_with(file_name) + else: + etag = f'"f20e84ac3d0c33cea77b3f29e3323a09-{parts}"' + m_checksum_file.return_value = digest + assert digest == s3.calculate_etag(module, file_name, etag, client, s3bucket_name, s3bucket_object, version) + + m_checksum_file.assert_called_with(client, parts, s3bucket_name, s3bucket_object, version, file_name) + + +@pytest.mark.parametrize("etag_multipart", [True, False]) +@patch("ansible_collections.amazon.aws.plugins.module_utils.s3.calculate_checksum_with_content") +def test_calculate_etag_content(m_checksum_content, etag_multipart): + module = MagicMock() + client = MagicMock() + + module.fail_json_aws.side_effect = SystemExit(2) + + s3bucket_name = f"s3-bucket-{generate_random_string(8, False)}" + s3bucket_object = f"s3-bucket-object-{generate_random_string(8, False)}" + version = random.randint(0, 1000) + parts = 3 + + etag = '"f20e84ac3d0c33cea77b3f29e3323a09"' + content = b'"f20e84ac3d0c33cea77b3f29e3323a09"' + digest = '"9aa254f7f76fd14435b21e9448525b99"' + + if not etag_multipart: + assert digest == s3.calculate_etag_content( + module, content, etag, client, s3bucket_name, s3bucket_object, version + ) + else: + etag = f'"f20e84ac3d0c33cea77b3f29e3323a09-{parts}"' + m_checksum_content.return_value = digest + result = s3.calculate_etag_content(module, content, etag, client, s3bucket_name, s3bucket_object, version) + assert result == digest + + m_checksum_content.assert_called_with(client, parts, s3bucket_name, s3bucket_object, version, content) + + +@pytest.mark.parametrize("using_file", [True, False]) +@patch("ansible_collections.amazon.aws.plugins.module_utils.s3.calculate_checksum_with_content") +@patch("ansible_collections.amazon.aws.plugins.module_utils.s3.calculate_checksum_with_file") +def test_calculate_etag_failure(m_checksum_file, m_checksum_content, using_file): + module = MagicMock() + client = MagicMock() + + module.fail_json_aws.side_effect = SystemExit(2) + + s3bucket_name = f"s3-bucket-{generate_random_string(8, False)}" + s3bucket_object = f"s3-bucket-object-{generate_random_string(8, False)}" + version = random.randint(0, 1000) + parts = 3 + + etag = f'"f20e84ac3d0c33cea77b3f29e3323a09-{parts}"' + content = "some content or file name" + + if using_file: + test_method = s3.calculate_etag + m_checksum_file.side_effect = raise_botoclient_exception() + else: + test_method = s3.calculate_etag_content + m_checksum_content.side_effect = raise_botoclient_exception() + + with pytest.raises(SystemExit): + test_method(module, content, etag, client, s3bucket_name, s3bucket_object, version) + module.fail_json_aws.assert_called() + + +@pytest.mark.parametrize( + "bucket_name,result", + [ + ("docexamplebucket1", None), + ("log-delivery-march-2020", None), + ("my-hosted-content", None), + ("docexamplewebsite.com", None), + ("www.docexamplewebsite.com", None), + ("my.example.s3.bucket", None), + ("doc", None), + ("doc_example_bucket", "invalid character(s) found in the bucket name"), + ("DocExampleBucket", "invalid character(s) found in the bucket name"), + ("doc-example-bucket-", "bucket names must begin and end with a letter or number"), + ( + "this.string.has.more.than.63.characters.so.it.should.not.passed.the.validated", + "the length of an S3 bucket cannot exceed 63 characters", + ), + ("my", "the length of an S3 bucket must be at least 3 characters"), + ], +) +def test_validate_bucket_name(bucket_name, result): + assert result == s3.validate_bucket_name(bucket_name) + + +mod_urlparse = "ansible_collections.amazon.aws.plugins.module_utils.s3.urlparse" + + +class UrlInfo: + def __init__(self, scheme=None, hostname=None, port=None): + self.hostname = hostname + self.scheme = scheme + self.port = port + + +@patch(mod_urlparse) +def test_is_fakes3_with_none_arg(m_urlparse): + m_urlparse.side_effect = SystemExit(1) + result = s3.is_fakes3(None) + assert not result + m_urlparse.assert_not_called() + + +@pytest.mark.parametrize( + "url,scheme,result", + [ + ("https://test-s3.amazon.com", "https", False), + ("fakes3://test-s3.amazon.com", "fakes3", True), + ("fakes3s://test-s3.amazon.com", "fakes3s", True), + ], +) +@patch(mod_urlparse) +def test_is_fakes3(m_urlparse, url, scheme, result): + m_urlparse.return_value = UrlInfo(scheme=scheme) + assert result == s3.is_fakes3(url) + m_urlparse.assert_called_with(url) + + +@pytest.mark.parametrize( + "url,urlinfo,endpoint", + [ + ( + "fakes3://test-s3.amazon.com", + {"scheme": "fakes3", "hostname": "test-s3.amazon.com"}, + {"endpoint": "http://test-s3.amazon.com:80", "use_ssl": False}, + ), + ( + "fakes3://test-s3.amazon.com:8080", + {"scheme": "fakes3", "hostname": "test-s3.amazon.com", "port": 8080}, + {"endpoint": "http://test-s3.amazon.com:8080", "use_ssl": False}, + ), + ( + "fakes3s://test-s3.amazon.com", + {"scheme": "fakes3s", "hostname": "test-s3.amazon.com"}, + {"endpoint": "https://test-s3.amazon.com:443", "use_ssl": True}, + ), + ( + "fakes3s://test-s3.amazon.com:9096", + {"scheme": "fakes3s", "hostname": "test-s3.amazon.com", "port": 9096}, + {"endpoint": "https://test-s3.amazon.com:9096", "use_ssl": True}, + ), + ], +) +@patch(mod_urlparse) +def test_parse_fakes3_endpoint(m_urlparse, url, urlinfo, endpoint): + m_urlparse.return_value = UrlInfo(**urlinfo) + result = s3.parse_fakes3_endpoint(url) + assert endpoint == result + m_urlparse.assert_called_with(url) + + +@pytest.mark.parametrize( + "url,scheme,use_ssl", + [ + ("https://test-s3-ceph.amazon.com", "https", True), + ("http://test-s3-ceph.amazon.com", "http", False), + ], +) +@patch(mod_urlparse) +def test_parse_ceph_endpoint(m_urlparse, url, scheme, use_ssl): + m_urlparse.return_value = UrlInfo(scheme=scheme) + result = s3.parse_ceph_endpoint(url) + assert result == {"endpoint": url, "use_ssl": use_ssl} + m_urlparse.assert_called_with(url) diff --git a/tests/unit/module_utils/test_tagging.py b/tests/unit/plugins/module_utils/test_tagging.py similarity index 100% rename from tests/unit/module_utils/test_tagging.py rename to tests/unit/plugins/module_utils/test_tagging.py diff --git a/tests/unit/module_utils/test_tower.py b/tests/unit/plugins/module_utils/test_tower.py similarity index 100% rename from tests/unit/module_utils/test_tower.py rename to tests/unit/plugins/module_utils/test_tower.py diff --git a/tests/unit/module_utils/transformation/__init__.py b/tests/unit/plugins/module_utils/transformation/__init__.py similarity index 100% rename from tests/unit/module_utils/transformation/__init__.py rename to tests/unit/plugins/module_utils/transformation/__init__.py diff --git a/tests/unit/module_utils/transformation/test_ansible_dict_to_boto3_filter_list.py b/tests/unit/plugins/module_utils/transformation/test_ansible_dict_to_boto3_filter_list.py similarity index 100% rename from tests/unit/module_utils/transformation/test_ansible_dict_to_boto3_filter_list.py rename to tests/unit/plugins/module_utils/transformation/test_ansible_dict_to_boto3_filter_list.py diff --git a/tests/unit/module_utils/transformation/test_boto3_resource_to_ansible_dict.py b/tests/unit/plugins/module_utils/transformation/test_boto3_resource_to_ansible_dict.py similarity index 100% rename from tests/unit/module_utils/transformation/test_boto3_resource_to_ansible_dict.py rename to tests/unit/plugins/module_utils/transformation/test_boto3_resource_to_ansible_dict.py diff --git a/tests/unit/module_utils/transformation/test_map_complex_type.py b/tests/unit/plugins/module_utils/transformation/test_map_complex_type.py similarity index 100% rename from tests/unit/module_utils/transformation/test_map_complex_type.py rename to tests/unit/plugins/module_utils/transformation/test_map_complex_type.py diff --git a/tests/unit/module_utils/transformation/test_sanitize_filters_to_boto3_filter_list.py b/tests/unit/plugins/module_utils/transformation/test_sanitize_filters_to_boto3_filter_list.py similarity index 100% rename from tests/unit/module_utils/transformation/test_sanitize_filters_to_boto3_filter_list.py rename to tests/unit/plugins/module_utils/transformation/test_sanitize_filters_to_boto3_filter_list.py diff --git a/tests/unit/module_utils/transformation/test_scrub_none_parameters.py b/tests/unit/plugins/module_utils/transformation/test_scrub_none_parameters.py similarity index 100% rename from tests/unit/module_utils/transformation/test_scrub_none_parameters.py rename to tests/unit/plugins/module_utils/transformation/test_scrub_none_parameters.py diff --git a/tests/unit/module_utils/waiter/test_custom_waiter_config.py b/tests/unit/plugins/module_utils/waiter/test_custom_waiter_config.py similarity index 100% rename from tests/unit/module_utils/waiter/test_custom_waiter_config.py rename to tests/unit/plugins/module_utils/waiter/test_custom_waiter_config.py diff --git a/tests/unit/plugins/modules/conftest.py b/tests/unit/plugins/modules/conftest.py index 7a870163c11..fb8e36bed58 100644 --- a/tests/unit/plugins/modules/conftest.py +++ b/tests/unit/plugins/modules/conftest.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright (c) 2017 Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) diff --git a/tests/unit/plugin_utils/__init__.py b/tests/unit/plugins/plugin_utils/__init__.py similarity index 100% rename from tests/unit/plugin_utils/__init__.py rename to tests/unit/plugins/plugin_utils/__init__.py diff --git a/tests/unit/plugin_utils/base/__init__.py b/tests/unit/plugins/plugin_utils/base/__init__.py similarity index 100% rename from tests/unit/plugin_utils/base/__init__.py rename to tests/unit/plugins/plugin_utils/base/__init__.py diff --git a/tests/unit/plugin_utils/base/test_plugin.py b/tests/unit/plugins/plugin_utils/base/test_plugin.py similarity index 100% rename from tests/unit/plugin_utils/base/test_plugin.py rename to tests/unit/plugins/plugin_utils/base/test_plugin.py diff --git a/tests/unit/plugin_utils/botocore/__init__.py b/tests/unit/plugins/plugin_utils/botocore/__init__.py similarity index 100% rename from tests/unit/plugin_utils/botocore/__init__.py rename to tests/unit/plugins/plugin_utils/botocore/__init__.py diff --git a/tests/unit/plugin_utils/botocore/test_boto3_conn_plugin.py b/tests/unit/plugins/plugin_utils/botocore/test_boto3_conn_plugin.py similarity index 100% rename from tests/unit/plugin_utils/botocore/test_boto3_conn_plugin.py rename to tests/unit/plugins/plugin_utils/botocore/test_boto3_conn_plugin.py diff --git a/tests/unit/plugin_utils/botocore/test_get_aws_region.py b/tests/unit/plugins/plugin_utils/botocore/test_get_aws_region.py similarity index 100% rename from tests/unit/plugin_utils/botocore/test_get_aws_region.py rename to tests/unit/plugins/plugin_utils/botocore/test_get_aws_region.py diff --git a/tests/unit/plugin_utils/botocore/test_get_connection_info.py b/tests/unit/plugins/plugin_utils/botocore/test_get_connection_info.py similarity index 100% rename from tests/unit/plugin_utils/botocore/test_get_connection_info.py rename to tests/unit/plugins/plugin_utils/botocore/test_get_connection_info.py diff --git a/tests/unit/plugin_utils/connection/__init__.py b/tests/unit/plugins/plugin_utils/connection/__init__.py similarity index 100% rename from tests/unit/plugin_utils/connection/__init__.py rename to tests/unit/plugins/plugin_utils/connection/__init__.py diff --git a/tests/unit/plugin_utils/connection/test_connection_base.py b/tests/unit/plugins/plugin_utils/connection/test_connection_base.py similarity index 100% rename from tests/unit/plugin_utils/connection/test_connection_base.py rename to tests/unit/plugins/plugin_utils/connection/test_connection_base.py diff --git a/tests/unit/plugin_utils/inventory/test_inventory_base.py b/tests/unit/plugins/plugin_utils/inventory/test_inventory_base.py similarity index 100% rename from tests/unit/plugin_utils/inventory/test_inventory_base.py rename to tests/unit/plugins/plugin_utils/inventory/test_inventory_base.py diff --git a/tests/unit/plugin_utils/inventory/test_inventory_clients.py b/tests/unit/plugins/plugin_utils/inventory/test_inventory_clients.py similarity index 100% rename from tests/unit/plugin_utils/inventory/test_inventory_clients.py rename to tests/unit/plugins/plugin_utils/inventory/test_inventory_clients.py diff --git a/tests/unit/plugin_utils/lookup/__init__.py b/tests/unit/plugins/plugin_utils/lookup/__init__.py similarity index 100% rename from tests/unit/plugin_utils/lookup/__init__.py rename to tests/unit/plugins/plugin_utils/lookup/__init__.py diff --git a/tests/unit/plugin_utils/lookup/test_lookup_base.py b/tests/unit/plugins/plugin_utils/lookup/test_lookup_base.py similarity index 100% rename from tests/unit/plugin_utils/lookup/test_lookup_base.py rename to tests/unit/plugins/plugin_utils/lookup/test_lookup_base.py diff --git a/tox.ini b/tox.ini index 0cdbe66516f..b0f3d58d0f5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,94 +1,216 @@ +# It would be nice to merge this into pyproject.toml, unfortunately as of 4.23.2 they don't support generative environments when using TOML + [tox] skipsdist = True -envlist = clean,ansible{2.12,2.13}-py{38,39,310}-{with_constraints,without_constraints},linters -# Tox4 supports labels which allow us to group the environments rather than dumping all commands into a single environment -labels = - format = flynt, black, isort - lint = complexity-report, ansible-lint, black-lint, isort-lint, flake8-lint, flynt-lint - units = ansible{2.12,2.13}-py{38,39,310}-{with_constraints,without_constraints} +skip_missing_interpreters = True +envlist = + ansible{2.15}-py{39,310,311}-{with_constraints,without_constraints} + ansible{2.16,2.17}-py{310,311,312}-{with_constraints,without_constraints} +# ansible{2.18}-py{311,312,313}-{with_constraints,without_constraints} [common] +collection_name = amazon.aws +collection_path = amazon/aws + format_dirs = {toxinidir}/plugins {toxinidir}/tests +lint_dirs = {toxinidir}/plugins {toxinidir}/tests + +ansible_desc = + ansible2.15: Ansible-core 2.15 + ansible2.16: Ansible-core 2.16 + ansible2.17: Ansible-core 2.17 + ansible2.18: Ansible-core 2.18 +const_desc = + with_constraints: (With boto3/botocore constraints) + +ansible_home = {envtmpdir}/ansible_home +ansible_collections_path = {[common]ansible_home}/collections +full_collection_path = {[common]ansible_home}/collections/ansible_collections/{[common]collection_path} [testenv] -description = Run the test-suite and generate a HTML coverage report +description = Run the unit tests {[common]ansible_desc}/{base_python} {[common]const_desc} +set_env = + ANSIBLE_HOME={[common]ansible_home} + ANSIBLE_COLLECTIONS_PATH={[common]ansible_collections_path} +# ansible_pytest_collections is more aggressive than pytest_ansible when injecting collections into the import path +# not needed if unit tests are under tests/unit/plugins rather than directly under tests/unit +# ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION=3.11 +# PYTEST_PLUGINS=ansible_test._util.target.pytest.plugins.ansible_pytest_collections +labels = unit deps = pytest + mock + pytest-mock pytest-cov - ansible2.12: ansible-core>2.12,<2.13 - ansible2.13: ansible-core>2.13,<2.14 - !ansible2.12-!ansible2.13: ansible-core pytest-ansible + pytest-xdist -rtest-requirements.txt + -rtests/unit/requirements.txt + ansible2.15: ansible-core>2.15,<2.16 + ansible2.16: ansible-core>2.16,<2.17 + ansible2.17: ansible-core>2.17,<2.18 + ansible2.18: ansible-core>2.18,<2.19 with_constraints: -rtests/unit/constraints.txt -commands = pytest --cov-report html --cov plugins/callback --cov plugins/inventory --cov plugins/lookup --cov plugins/module_utils --cov plugins/modules --cov plugins/plugin_utils plugins {posargs:tests/} +allowlist_externals = rsync +change_dir = {[common]full_collection_path} +commands_pre = + rsync --delete --exclude=.tox -qraugpo {toxinidir}/ {[common]full_collection_path}/ + ansible-galaxy collection install git+https://github.com/ansible-collections/community.aws.git,stable-9 +commands = + pytest \ + --cov-report html \ + --cov plugins/callback \ + --cov plugins/inventory \ + --cov plugins/lookup \ + --cov plugins/module_utils \ + --cov plugins/modules \ + --cov plugins/plugin_utils \ + --cov plugins \ + --ansible-host-pattern localhost \ + {posargs:tests/unit/} [testenv:clean] +description = Remove test results and caches +allowlist_externals = rm deps = coverage skip_install = true -commands = coverage erase +change_dir = {toxinidir} +commands_pre = +commands = + coverage erase + rm -rf tests/output/ htmlcov/ .mypy_cache/ complexity/ .ruff_cache/ [testenv:complexity-report] +labels = future-lint description = Generate a HTML complexity report in the complexity directory deps = - # See: https://github.com/lordmauve/flake8-html/issues/30 - flake8>=3.3.0,<5.0.0 + flake8-pyproject flake8-html -commands = -flake8 --select C90 --max-complexity 10 --format=html --htmldir={posargs:complexity} plugins +change_dir = {toxinidir} +commands_pre = +commands = + -flake8 \ + --select C90 \ + --max-complexity 10 \ + --format=html \ + --htmldir={posargs:complexity} \ + {posargs:{[common]lint_dirs}} [testenv:ansible-lint] +labels = lint-future +description = Run ansible-lint deps = ansible-lint >= 24.7.0 + jmespath +change_dir = {toxinidir} +commands_pre = commands = - ansible-lint {toxinidir}/plugins + ansible-lint \ + --skip-list=name[missing],yaml[line-length],args[module],run-once[task],ignore-errors,sanity[cannot-ignore],run-once[play] \ + {posargs:{[common]lint_dirs}} [testenv:black] +labels = format +description = Apply "black" formatting depends = flynt, isort deps = black >=23.0, <24.0 +change_dir = {toxinidir} +commands_pre = commands = - black {[common]format_dirs} + black {posargs:{[common]format_dirs}} [testenv:black-lint] +labels = lint +description = Lint against "black" formatting standards deps = {[testenv:black]deps} +change_dir = {toxinidir} +commands_pre = commands = - black -v --check --diff {[common]format_dirs} + black --check --diff {posargs:{[common]format_dirs}} [testenv:isort] +labels = format +description = Sort imports deps = isort +change_dir = {toxinidir} +commands_pre = commands = - isort {[common]format_dirs} + isort {posargs:{[common]format_dirs}} [testenv:isort-lint] +labels = lint +description = Lint for import sorting deps = {[testenv:isort]deps} +change_dir = {toxinidir} +commands_pre = +commands = + isort --check-only --diff {posargs:{[common]format_dirs}} + +[testenv:flynt] +labels = format +description = Apply flint (f-string) formatting +deps = + flynt +change_dir = {toxinidir} +commands_pre = commands = - isort --check-only --diff {[common]format_dirs} + flynt {posargs:{[common]format_dirs}} + +[testenv:flynt-lint] +labels = lint +description = Run flint (f-string) linting +deps = + flynt +change_dir = {toxinidir} +commands_pre = +commands = + flynt --dry-run --fail-on-change {posargs:{[common]format_dirs}} [testenv:flake8-lint] +labels = lint +description = Run FLAKE8 linting deps = flake8 + flake8-pyproject +change_dir = {toxinidir} +commands_pre = commands = - flake8 {posargs} {[common]format_dirs} + flake8 {posargs:{[common]format_dirs}} [testenv:pylint-lint] -# Additional pylint tests that ansible-test currently ignores +labels = lint-future +description = Run pylint tests that are disabled by the default Ansible sanity tests deps = pylint +change_dir = {toxinidir} +commands_pre = commands = pylint \ --disable R,C,W,E \ - --enable consider-using-dict-items,assignment-from-no-return,no-else-continue,no-else-break,simplifiable-if-statement,pointless-string-statement,redefined-outer-name,redefined-builtin \ - {toxinidir}/plugins + --enable pointless-statement \ + --enable consider-using-dict-items \ + --enable assignment-from-no-return \ + --enable no-else-continue \ + --enable no-else-break \ + --enable simplifiable-if-statement \ + --enable pointless-string-statement \ + --enable redefined-outer-name \ + --enable redefined-builtin \ + --enable unused-import \ + {posargs:{[common]lint_dirs}} [testenv:ruff] description = lint source code labels = format-future deps = ruff +change_dir = {toxinidir} +commands_pre = commands = ruff check --fix {posargs:{[common]lint_dirs}} ruff format {posargs:{[common]lint_dirs}} @@ -98,55 +220,60 @@ description = lint source code labels = lint-future deps = ruff +change_dir = {toxinidir} +commands_pre = commands = ruff check --diff --unsafe-fixes {posargs:{[common]lint_dirs}} ruff check {posargs:{[common]lint_dirs}} [testenv:ansible-lint-future] -allowlist_externals = echo,cd,rm,mkdir,ln,ls labels = future-lint description = Run ansible-lint -# ansible-lint expects us to be installed into ansible_collections/amazon/aws -# by default we're checked out into amazon.aws -set_env = - ANSIBLE_HOME={[future-lint]lint_tmp_path} -commands_pre = - rm -rf {[future-lint]lint_tmp_path} - mkdir -p {[future-lint]full_tmp_path} - rm -d {[future-lint]full_tmp_path} - ln -s {toxinidir} {[future-lint]full_tmp_path} - ansible-galaxy collection install git+https://github.com/ansible-collections/community.aws.git - ansible-galaxy collection install -r tests/integration/requirements.yml deps = - flynt + ansible-lint + jmespath + git+https://github.com/ansible/ansible.git@devel + shellcheck-py commands = - flynt {[common]format_dirs} - -[testenv:flynt-lint] -deps = - flynt -commands = - flynt --dry-run --fail-on-change {[common]format_dirs} - -[testenv:linters] -deps = - {[testenv:black]deps} - {[testenv:isort]deps} - flake8 -commands = - black -v --check {toxinidir}/plugins {toxinidir}/tests - isort --check-only --diff {toxinidir}/plugins {toxinidir}/tests - flake8 {posargs} {toxinidir}/plugins {toxinidir}/tests + ansible-lint \ + {posargs:{[common]lint_dirs}} [testenv:ansible-sanity] +labels = future-lint +description = Run latest (devel) Ansible sanity tests deps = - git+https://github.com/ansible/ansible.git@milestone + git+https://github.com/ansible/ansible.git@devel + shellcheck-py commands = ansible-test sanity -[flake8] -# E123, E125 skipped as they are invalid PEP-8. -show-source = True -ignore = E123,E125,E203,E402,E501,E741,F401,F811,F841,W503 -max-line-length = 160 -builtins = _ +[testenv:mypy-lint] +allowlist_externals = rsync,ln +labels = future-lint +description = Run mypi type tests +set_env = + ANSIBLE_HOME={[common]ansible_home} + ANSIBLE_COLLECTIONS_PATH={[common]ansible_collections_path} + MYPYPATH={[common]ansible_home} +deps = + mypy + # ansible-core + git+https://github.com/ansible/ansible.git@devel + botocore + boto3 + placebo + typing_extensions +commands_pre = + rsync --delete --exclude=.tox -qraugpo {toxinidir}/ {[common]full_collection_path}/ + ansible-galaxy collection install git+https://github.com/ansible-collections/community.aws.git,stable-9 + ln -s {env_site_packages_dir}/ansible {[common]ansible_collections_path}/ansible + ln -s {[common]ansible_home}/collections/ansible_collections {[common]ansible_home}/ansible_collections +commands = + # TODO: passing directories doesn't work well, it's better to pass the package + # we might want to consider manipulating posargs/directories into the package names + mypy \ + --check-untyped-defs \ + --namespace-packages --explicit-package-bases \ + --follow-imports silent \ + -p ansible_collections.amazon.aws.plugins.plugin_utils \ + -p ansible_collections.amazon.aws.plugins.module_utils