From 7aa5b832cb7cb51379b186646d5018f1a1e756cb Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 27 Sep 2023 22:00:24 +0200 Subject: [PATCH] [7.0.0] Bump minimum botocore requirements (#1763) [7.0.0] Bump minimum botocore requirements SUMMARY In line with policy bump minimum botocore requirement to 1.29.0 (released Nov 2022) in preparation for release 7.0.0 ISSUE TYPE Feature Pull Request COMPONENT NAME plugins/module_utils/botocore.py ADDITIONAL INFORMATION Reviewed-by: Alina Buzachis --- README.md | 30 ++++++++++++++----- changelogs/fragments/botocore.yml | 6 ++++ changelogs/fragments/python37.yml | 10 +++++++ plugins/module_utils/botocore.py | 4 +-- plugins/modules/backup_restore_job_info.py | 2 -- plugins/modules/ec2_ami.py | 4 --- plugins/modules/rds_cluster.py | 1 - plugins/modules/rds_instance.py | 18 +++++------ requirements.txt | 4 +-- tests/config.yml | 2 +- tests/integration/constraints.txt | 6 ++-- .../targets/aws_az_info/tasks/main.yml | 30 +++++++------------ .../integration/targets/ec2_ami/meta/main.yml | 3 -- .../targets/ec2_ami/tasks/main.yml | 10 ------- .../targets/ec2_ami_tpm/meta/main.yml | 3 -- .../targets/ec2_ami_tpm/tasks/main.yml | 4 --- .../rds_cluster_multi_az/meta/main.yml | 5 +--- .../rds_cluster_multi_az/tasks/main.yml | 4 --- tests/unit/constraints.txt | 6 ++-- .../test_minimal_versions.py | 8 ++--- tests/unit/plugins/modules/test_ec2_ami.py | 4 +-- 21 files changed, 73 insertions(+), 91 deletions(-) create mode 100644 changelogs/fragments/botocore.yml create mode 100644 changelogs/fragments/python37.yml diff --git a/README.md b/README.md index b3fdfe222b3..86c25304cfb 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,33 @@ Use amazon.aws 4.x.y if you are using Ansible 2.9 or Ansible Core 2.10. This collection depends on the AWS SDK for Python (Boto3 and Botocore). Due to the [AWS SDK Python Support Policy](https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/) -this collection requires Python 3.6 or greater. - -Amazon have also announced the end of support for -[Python less than 3.7](https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/). -As such support for Python less than 3.7 by this collection has been deprecated and will be removed in release 7.0.0. -Additionally, support for Python less than 3.8 is expected to be removed in a release after 2024-12-01 based on currently -available schedules. +this collection requires Python 3.7 or greater. + +Amazon have also announced the planned end of support for +[Python less than 3.8](https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/). +As such support for Python less than 3.8 will be removed in a release after 2024-12-01. + + ## AWS SDK version compatibility Starting with the 2.0.0 releases of amazon.aws and community.aws, it is generally the collection's policy to support the versions of `botocore` and `boto3` that were released 12 months prior to the most recent major collection release, following semantic versioning (for example, 2.0.0, 3.0.0). -Version 6.0.0 of this collection supports `boto3 >= 1.22.0` and `botocore >= 1.25.0` +Version 7.0.0 of this collection supports `boto3 >= 1.26.0` and `botocore >= 1.29.0` All support for the original AWS SDK `boto` was removed in release 4.0.0. diff --git a/changelogs/fragments/botocore.yml b/changelogs/fragments/botocore.yml new file mode 100644 index 00000000000..9defbf18cb9 --- /dev/null +++ b/changelogs/fragments/botocore.yml @@ -0,0 +1,6 @@ +breaking_changes: +- The amazon.aws collection has dropped support for ``botocore<1.29.0`` and + ``boto3<1.26.0``. Most modules will continue to work with older versions of the AWS SDK, however + compatability with older versions of the SDK is not guaranteed and will not be tested. When using + older versions of the SDK a warning will be emitted by Ansible + (https://github.com/ansible-collections/amazon.aws/pull/1763). diff --git a/changelogs/fragments/python37.yml b/changelogs/fragments/python37.yml new file mode 100644 index 00000000000..d061bd466be --- /dev/null +++ b/changelogs/fragments/python37.yml @@ -0,0 +1,10 @@ +breaking_changes: +- amazon.aws collection - due to the AWS SDKs announcing the end of support + for Python less than 3.7 (https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/) + support for Python less than 3.7 by this collection wss been deprecated in release 6.0.0 and removed in release 7.0.0. + (https://github.com/ansible-collections/amazon.aws/pull/1763). + +# We've already announced the deprecation for <3.8 (with 6.0.0), dropping support for <3.9 on ours side will happen +# after April 2026. This is about 2 years + 5 months away assuming a November 7.0.0 release, we could announce +# the deprecation now, but assuming we release 8.0.0 in about 6 months a just short of 2 year +# deprecation feels fine given it's predictable. diff --git a/plugins/module_utils/botocore.py b/plugins/module_utils/botocore.py index c04b27d53a9..1c3f2b5dd7e 100644 --- a/plugins/module_utils/botocore.py +++ b/plugins/module_utils/botocore.py @@ -64,8 +64,8 @@ from .retries import AWSRetry from .common import get_collection_info -MINIMUM_BOTOCORE_VERSION = "1.25.0" -MINIMUM_BOTO3_VERSION = "1.22.0" +MINIMUM_BOTOCORE_VERSION = "1.29.0" +MINIMUM_BOTO3_VERSION = "1.26.0" def _get_user_agent_string(): diff --git a/plugins/modules/backup_restore_job_info.py b/plugins/modules/backup_restore_job_info.py index e11a8d4cc54..c6ed71e7a26 100644 --- a/plugins/modules/backup_restore_job_info.py +++ b/plugins/modules/backup_restore_job_info.py @@ -214,8 +214,6 @@ def main(): backup_client = module.client("backup") - module.require_botocore_at_least("1.25.13", reason="to list restore jobs info") - request_args = build_request_args( account_id=module.params["account_id"], status=module.params["status"], diff --git a/plugins/modules/ec2_ami.py b/plugins/modules/ec2_ami.py index d889f5c47ad..000469ee6a3 100644 --- a/plugins/modules/ec2_ami.py +++ b/plugins/modules/ec2_ami.py @@ -558,10 +558,6 @@ def validate_params( if not (image_id or name): module.fail_json("one of the following is required: name, image_id") - if tpm_support or uefi_data: - module.require_botocore_at_least( - "1.26.0", reason="required for ec2.register_image with tpm_support or uefi_data" - ) if tpm_support and boot_mode != "uefi": module.fail_json("To specify 'tpm_support', 'boot_mode' must be 'uefi'.") diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 187bfbe28e6..6ac90fd0f0d 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1206,7 +1206,6 @@ def main(): module.fail_json_aws(e, msg="Failed to connect to AWS.") if module.params.get("engine") and module.params["engine"] in ("mysql", "postgres"): - module.require_botocore_at_least("1.23.44", reason="to use mysql and postgres engines") if module.params["state"] == "present": if not ( module.params.get("allocated_storage") diff --git a/plugins/modules/rds_instance.py b/plugins/modules/rds_instance.py index 2f6124a5199..14366678807 100644 --- a/plugins/modules/rds_instance.py +++ b/plugins/modules/rds_instance.py @@ -1055,17 +1055,13 @@ def get_options_with_changing_values(client, module, parameters): parameters["Iops"] = new_iops if instance.get("StorageType") == "gp3": - if module.botocore_at_least("1.29.0"): - GP3_THROUGHPUT = True - current_storage_throughput = instance.get("PendingModifiedValues", {}).get( - "StorageThroughput", instance["StorageThroughput"] - ) - new_storage_throughput = module.params.get("storage_throughput") or current_storage_throughput - if new_storage_throughput != current_storage_throughput: - parameters["StorageThroughput"] = new_storage_throughput - else: - GP3_THROUGHPUT = False - module.warn("gp3 volumes require botocore >= 1.29.0. storage_throughput will be ignored.") + GP3_THROUGHPUT = True + current_storage_throughput = instance.get("PendingModifiedValues", {}).get( + "StorageThroughput", instance["StorageThroughput"] + ) + new_storage_throughput = module.params.get("storage_throughput") or current_storage_throughput + if new_storage_throughput != current_storage_throughput: + parameters["StorageThroughput"] = new_storage_throughput current_iops = instance.get("PendingModifiedValues", {}).get("Iops", instance["Iops"]) # when you just change from gp2 to gp3, you may not add the iops parameter diff --git a/requirements.txt b/requirements.txt index 4853d7e0c7c..cd474e3b66b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ # - tests/unit/constraints.txt # - tests/integration/constraints.txt # - tests/integration/targets/setup_botocore_pip -botocore>=1.25.0 -boto3>=1.22.0 +botocore>=1.29.0 +boto3>=1.26.0 diff --git a/tests/config.yml b/tests/config.yml index 5112f726881..8d053169d67 100644 --- a/tests/config.yml +++ b/tests/config.yml @@ -1,2 +1,2 @@ modules: - python_requires: '>=3.6' + python_requires: '>=3.7' diff --git a/tests/integration/constraints.txt b/tests/integration/constraints.txt index 73cb8913ddb..f388e1f900b 100644 --- a/tests/integration/constraints.txt +++ b/tests/integration/constraints.txt @@ -1,10 +1,10 @@ # Specifically run tests against the oldest versions that we support -boto3==1.22.0 -botocore==1.25.0 +botocore==1.29.0 +boto3==1.26.0 # AWS CLI has `botocore==` dependencies, provide the one that matches botocore # to avoid needing to download over a years worth of awscli wheels. -awscli==1.23.0 +awscli==1.27.0 # AWS CLI depends on PyYAML <5.5,>=3.10; the latest PyYAML release in that range, 5.4.1, fails to install. # Use a version in that range that is known to work (https://github.com/yaml/pyyaml/issues/736) diff --git a/tests/integration/targets/aws_az_info/tasks/main.yml b/tests/integration/targets/aws_az_info/tasks/main.yml index 1ae90be1b54..934e17c46ac 100644 --- a/tests/integration/targets/aws_az_info/tasks/main.yml +++ b/tests/integration/targets/aws_az_info/tasks/main.yml @@ -26,8 +26,7 @@ - '"state" in first_az' - '"zone_id" in first_az' - '"zone_name" in first_az' - # botocore >= 1.17.18 - #- '"zone_type" in first_az' + - '"zone_type" in first_az' - name: 'List available AZs in current Region - check_mode' aws_az_info: @@ -49,8 +48,7 @@ - '"state" in first_az' - '"zone_id" in first_az' - '"zone_name" in first_az' - # botocore >= 1.17.18 - #- '"zone_type" in first_az' + - '"zone_type" in first_az' # Be specific - aws_region isn't guaranteed to be any specific value @@ -74,8 +72,7 @@ - '"state" in first_az' - '"zone_id" in first_az' - '"zone_name" in first_az' - # botocore >= 1.17.18 - #- '"zone_type" in first_az' + - '"zone_type" in first_az' - first_az.group_name.startswith('us-east-1') - first_az.network_border_group.startswith('us-east-1') - first_az.region_name == 'us-east-1' @@ -83,8 +80,7 @@ - not first_az.zone_id == "use1-az" - first_az.zone_name.startswith('us-east-1') - not first_az.zone_name == 'us-east-1' - # botocore >= 1.17.18 - #- first_az.zone_type == 'availability-zone' + - first_az.zone_type == 'availability-zone' - name: 'Filter Available AZs in us-west-2 using - ("zone-name")' aws_az_info: @@ -109,8 +105,7 @@ - '"state" in first_az' - '"zone_id" in first_az' - '"zone_name" in first_az' - # botocore >= 1.17.18 - #- '"zone_type" in first_az' + - '"zone_type" in first_az' - first_az.group_name == 'us-west-2' - first_az.network_border_group == 'us-west-2' - first_az.region_name == 'us-west-2' @@ -118,8 +113,7 @@ - first_az.zone_id.startswith('usw2-az') - not first_az.zone_id == 'usw2-az' - first_az.zone_name == 'us-west-2c' - # botocore >= 1.17.18 - #- first_az.zone_type == 'availability-zone' + - first_az.zone_type == 'availability-zone' - name: 'Filter Available AZs in eu-central-1 using _ ("zone_name")' aws_az_info: @@ -144,8 +138,7 @@ - '"state" in first_az' - '"zone_id" in first_az' - '"zone_name" in first_az' - # botocore >= 1.17.18 - #- '"zone_type" in first_az' + - '"zone_type" in first_az' - first_az.group_name == 'eu-central-1' - first_az.network_border_group == 'eu-central-1' - first_az.region_name == 'eu-central-1' @@ -153,8 +146,7 @@ - first_az.zone_id.startswith('euc1-az') - not first_az.zone_id == "euc1-az" - first_az.zone_name == 'eu-central-1b' - # botocore >= 1.17.18 - #- first_az.zone_type == 'availability-zone' + - first_az.zone_type == 'availability-zone' - name: 'Filter Available AZs in eu-west-2 using _ and - ("zone_name" and "zone-name") : _ wins ' aws_az_info: @@ -180,8 +172,7 @@ - '"state" in first_az' - '"zone_id" in first_az' - '"zone_name" in first_az' - # botocore >= 1.17.18 - #- '"zone_type" in first_az' + - '"zone_type" in first_az' - first_az.group_name == 'eu-west-2' - first_az.network_border_group == 'eu-west-2' - first_az.region_name == 'eu-west-2' @@ -189,5 +180,4 @@ - first_az.zone_id.startswith('euw2-az') - not first_az.zone_id == "euw2-az" - first_az.zone_name == 'eu-west-2c' - # botocore >= 1.17.18 - #- first_az.zone_type == 'availability-zone' + - first_az.zone_type == 'availability-zone' diff --git a/tests/integration/targets/ec2_ami/meta/main.yml b/tests/integration/targets/ec2_ami/meta/main.yml index 3dc000aba61..1471b11f658 100644 --- a/tests/integration/targets/ec2_ami/meta/main.yml +++ b/tests/integration/targets/ec2_ami/meta/main.yml @@ -1,5 +1,2 @@ dependencies: - setup_ec2_facts - - role: setup_botocore_pip - vars: - botocore_version: '1.26.0' \ No newline at end of file diff --git a/tests/integration/targets/ec2_ami/tasks/main.yml b/tests/integration/targets/ec2_ami/tasks/main.yml index 6d61e0b1f94..7b3f551d5d5 100644 --- a/tests/integration/targets/ec2_ami/tasks/main.yml +++ b/tests/integration/targets/ec2_ami/tasks/main.yml @@ -10,12 +10,6 @@ - amazon.aws block: - # AWS CLI is needed until there's a module to get instance uefi data - - name: Install AWS CLI - pip: - name: awscli==1.25.83 - state: present - # ============================================================ # SETUP: vpc, ec2 key pair, subnet, security group, ec2 instance, snapshot @@ -672,8 +666,6 @@ snapshot_id: '{{ setup_snapshot.snapshot_id }}' register: result ignore_errors: true - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - name: set image id fact for deletion later set_fact: @@ -685,8 +677,6 @@ image_ids: '{{ ec2_ami_image_id_boot_tpm }}' register: ami_facts_result_boot_tpm ignore_errors: true - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - name: assert that new ami has been created with desired options assert: diff --git a/tests/integration/targets/ec2_ami_tpm/meta/main.yml b/tests/integration/targets/ec2_ami_tpm/meta/main.yml index 3dc000aba61..1471b11f658 100644 --- a/tests/integration/targets/ec2_ami_tpm/meta/main.yml +++ b/tests/integration/targets/ec2_ami_tpm/meta/main.yml @@ -1,5 +1,2 @@ dependencies: - setup_ec2_facts - - role: setup_botocore_pip - vars: - botocore_version: '1.26.0' \ No newline at end of file diff --git a/tests/integration/targets/ec2_ami_tpm/tasks/main.yml b/tests/integration/targets/ec2_ami_tpm/tasks/main.yml index 092f72a5574..8fb698d42c4 100644 --- a/tests/integration/targets/ec2_ami_tpm/tasks/main.yml +++ b/tests/integration/targets/ec2_ami_tpm/tasks/main.yml @@ -96,8 +96,6 @@ snapshot_id: '{{ setup_snapshot.snapshot_id }}' register: result ignore_errors: true - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - name: set image id fact for deletion later set_fact: @@ -109,8 +107,6 @@ image_ids: '{{ ec2_ami_image_id_boot_tpm }}' register: ami_facts_result_boot_tpm ignore_errors: true - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - name: assert that new ami has been created with desired options assert: diff --git a/tests/integration/targets/rds_cluster_multi_az/meta/main.yml b/tests/integration/targets/rds_cluster_multi_az/meta/main.yml index 39e88928aaa..23d65c7ef45 100644 --- a/tests/integration/targets/rds_cluster_multi_az/meta/main.yml +++ b/tests/integration/targets/rds_cluster_multi_az/meta/main.yml @@ -1,5 +1,2 @@ --- -dependencies: - - role: setup_botocore_pip - vars: - botocore_version: "1.23.44" +dependencies: [] diff --git a/tests/integration/targets/rds_cluster_multi_az/tasks/main.yml b/tests/integration/targets/rds_cluster_multi_az/tasks/main.yml index 1d33be4ee3c..b1d0804101a 100644 --- a/tests/integration/targets/rds_cluster_multi_az/tasks/main.yml +++ b/tests/integration/targets/rds_cluster_multi_az/tasks/main.yml @@ -39,8 +39,6 @@ tags: '{{ tags_create }}' register: _result_create_source_db_cluster check_mode: True - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - assert: that: @@ -60,8 +58,6 @@ wait: true tags: '{{ tags_create }}' register: _result_create_source_db_cluster - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - assert: that: diff --git a/tests/unit/constraints.txt b/tests/unit/constraints.txt index 927a6f07b42..5708323f110 100644 --- a/tests/unit/constraints.txt +++ b/tests/unit/constraints.txt @@ -1,7 +1,7 @@ # Specifically run tests against the oldest versions that we support -boto3==1.22.0 -botocore==1.25.0 +botocore==1.29.0 +boto3==1.26.0 # AWS CLI has `botocore==` dependencies, provide the one that matches botocore # to avoid needing to download over a years worth of awscli wheels. -awscli==1.23.0 +awscli==1.27.0 diff --git a/tests/unit/module_utils/modules/ansible_aws_module/test_minimal_versions.py b/tests/unit/module_utils/modules/ansible_aws_module/test_minimal_versions.py index 79cc16ded30..15aa71eae9e 100644 --- a/tests/unit/module_utils/modules/ansible_aws_module/test_minimal_versions.py +++ b/tests/unit/module_utils/modules/ansible_aws_module/test_minimal_versions.py @@ -25,10 +25,10 @@ class TestMinimalVersionTestSuite: # Prepare some data for use in our testing # ======================================================== def setup_method(self): - self.MINIMAL_BOTO3 = "1.22.0" - self.MINIMAL_BOTOCORE = "1.25.0" - self.OLD_BOTO3 = "1.21.999" - self.OLD_BOTOCORE = "1.24.999" + self.MINIMAL_BOTO3 = "1.26.0" + self.MINIMAL_BOTOCORE = "1.29.0" + self.OLD_BOTO3 = "1.25.999" + self.OLD_BOTOCORE = "1.28.999" # ======================================================== # Test we don't warn when using valid versions diff --git a/tests/unit/plugins/modules/test_ec2_ami.py b/tests/unit/plugins/modules/test_ec2_ami.py index c051e4020a2..2ac59d9c427 100644 --- a/tests/unit/plugins/modules/test_ec2_ami.py +++ b/tests/unit/plugins/modules/test_ec2_ami.py @@ -137,11 +137,11 @@ def test_validate_params(): module = MagicMock() ec2_ami.validate_params(module, tpm_support=True) - assert module.require_botocore_at_least.call_count == 1 + assert module.require_botocore_at_least.call_count == 0 module = MagicMock() ec2_ami.validate_params(module, tpm_support=True, boot_mode="legacy-bios") - assert module.require_botocore_at_least.call_count == 1 + assert module.require_botocore_at_least.call_count == 0 module.fail_json.assert_any_call("To specify 'tpm_support', 'boot_mode' must be 'uefi'.") module = MagicMock()