chore: Upgrade Docker-py/ Support Py3.11 for running tests (aws#5279)

* fix: add 3.11 to classifiers and upgrade Docker (aws#5225)

* fix: add 3.11 to classifiers

- update dependencies, need to nail down the versions.

* Pin dev dependencies and handle excluding folders for mypy

* Remove unneeded type: ignores

* Fix name-match mypy errors

* Fix empty-body error from mypy

* Fix mypy errors by ignoring and get pytest to run/pass

* Force mypy to not fail hopefully

* Remove unneeded assignment

* Update pinned requirements file

---------

Co-authored-by: Jacob Fuss <jfuss@users.noreply.github.com>
Co-authored-by: Jacob Fuss <32497805+jfuss@users.noreply.github.com>

* chore: Force version on docker and allow unit test to run when docker not running

In order for the docker.from_env() not to fail when docker is not installed/running,
we force the min version on client creation. This was the default behavior in 4.X of
docker-py but not longer in the latest version.

---------

Co-authored-by: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com>
Co-authored-by: Jacob Fuss <jfuss@users.noreply.github.com>

Makefile

@@ -36,7 +36,7 @@ lint:
 	# Linter performs static analysis to catch latent bugs
 	ruff samcli
 	# mypy performs type check
-	mypy --no-incremental setup.py samcli tests
+	mypy --exclude /testdata/ --exclude /init/templates/ --no-incremental setup.py samcli tests
 
 # Command to run everytime you make changes to verify everything works
 dev: lint test

mypy.ini

@@ -5,7 +5,7 @@ warn_return_any=True
 warn_unused_configs=True
 no_implicit_optional=True
 warn_redundant_casts=True
-warn_unused_ignores=True
+warn_unused_ignores=False # @jfuss Done as a stop gap since different py versions have different errors
 warn_unreachable=True
 
 #

requirements/base.txt

@@ -3,21 +3,21 @@ click~=8.0
 Flask<2.3
 #Need to add Schemas latest SDK.
 boto3>=1.19.5,==1.*
-jmespath~=0.10.0
-ruamel_yaml==0.17.21
+jmespath~=1.0.1
+ruamel_yaml~=0.17.21
 PyYAML>=5.4.1,==5.*
 cookiecutter~=2.1.1
 aws-sam-translator==1.68.0
 #docker minor version updates can include breaking changes. Auto update micro version only.
-docker~=4.2.0
+docker~=6.1.0
 dateparser~=1.1
-requests==2.31.0
+requests~=2.31.0
 serverlessrepo==0.1.10
 aws_lambda_builders==1.33.0
 tomlkit==0.11.8
 watchdog==2.1.2
 rich~=13.3.3
-pyopenssl==23.0.0
+pyopenssl~=23.0.0
 
 # Needed for supporting Protocol in Python 3.7, Protocol class became public with python3.8
 typing_extensions~=4.4.0

requirements/dev.txt

@@ -1,22 +1,30 @@
 -r pre-dev.txt
 
-coverage==5.3
+coverage==7.2.7
 pytest-cov==4.0.0
 
 
 # type checking and related stubs
 # mypy adds new rules in new minor versions, which could cause our PR check to fail
 # here we fix its version and upgrade it manually in the future
-mypy==0.790
+mypy==1.3.0
 boto3-stubs[apigateway,cloudformation,ecr,iam,lambda,s3,schemas,secretsmanager,signer,stepfunctions,sts,xray]==1.26.131
 types-pywin32==306.0.0.0
 types-PyYAML==6.0.12
 types-chevron==0.14.2.4
 types-psutil==5.9.5.12
 types-setuptools==65.4.0.0
+types-Pygments==2.15.0.1 
+types-colorama==0.4.15.11
+types-dateparser==1.1.4.9
+types-docutils==0.20.0.1
+types-jsonschema==4.17.0.8
+types-pyOpenSSL==23.2.0.0
+types-requests==2.31.0.1
+types-urllib3==1.26.25.13
 
 # Test requirements
-pytest==7.2.2
+pytest~=7.2.2
 parameterized==0.9.0
 pytest-xdist==3.2.0
 pytest-forked==1.6.0

requirements/reproducible-linux.txt

@@ -241,9 +241,9 @@ dateparser==1.1.8 \
     --hash=sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f \
     --hash=sha256:86b8b7517efcc558f085a142cdb7620f0921543fcabdb538c8a4c4001d8178e3
     # via aws-sam-cli (setup.py)
-docker==4.2.2 \
-    --hash=sha256:03a46400c4080cb6f7aa997f881ddd84fef855499ece219d75fbdb53289c17ab \
-    --hash=sha256:26eebadce7e298f55b76a88c4f8802476c5eaddbdbe38dbc6cce8781c47c9b54
+docker==6.1.3 \
+    --hash=sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20 \
+    --hash=sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9
     # via aws-sam-cli (setup.py)
 flask==2.2.5 \
     --hash=sha256:58107ed83443e86067e41eff4631b058178191a355886f8e479e347fa1285fdf \
@@ -268,9 +268,9 @@ jinja2-time==0.2.0 \
     --hash=sha256:d14eaa4d315e7688daa4969f616f226614350c48730bfa1692d2caebd8c90d40 \
     --hash=sha256:d3eab6605e3ec8b7a0863df09cc1d23714908fa61aa6986a845c20ba488b4efa
     # via cookiecutter
-jmespath==0.10.0 \
-    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
-    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f
+jmespath==1.0.1 \
+    --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \
+    --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe
     # via
     #   aws-sam-cli (setup.py)
     #   boto3
@@ -371,6 +371,10 @@ networkx==2.6.3 \
     --hash=sha256:80b6b89c77d1dfb64a4c7854981b60aeea6360ac02c6d4e4913319e0a313abef \
     --hash=sha256:c0946ed31d71f1b732b5aaa6da5a0388a345019af232ce2f49c766e2d6795c51
     # via cfn-lint
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via docker
 pbr==5.11.1 \
     --hash=sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b \
     --hash=sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3
@@ -630,7 +634,6 @@ six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
-    #   docker
     #   junit-xml
     #   python-dateutil
     #   serverlessrepo
@@ -664,6 +667,7 @@ urllib3==1.26.15 \
     --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42
     # via
     #   botocore
+    #   docker
     #   requests
 watchdog==2.1.2 \
     --hash=sha256:0237db4d9024859bea27d0efb59fe75eef290833fd988b8ead7a879b0308c2db \

requirements/reproducible-mac.txt

@@ -259,9 +259,9 @@ dateparser==1.1.8 \
     --hash=sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f \
     --hash=sha256:86b8b7517efcc558f085a142cdb7620f0921543fcabdb538c8a4c4001d8178e3
     # via aws-sam-cli (setup.py)
-docker==4.2.2 \
-    --hash=sha256:03a46400c4080cb6f7aa997f881ddd84fef855499ece219d75fbdb53289c17ab \
-    --hash=sha256:26eebadce7e298f55b76a88c4f8802476c5eaddbdbe38dbc6cce8781c47c9b54
+docker==6.1.3 \
+    --hash=sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20 \
+    --hash=sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9
     # via aws-sam-cli (setup.py)
 flask==2.2.5 \
     --hash=sha256:58107ed83443e86067e41eff4631b058178191a355886f8e479e347fa1285fdf \
@@ -299,9 +299,9 @@ jinja2-time==0.2.0 \
     --hash=sha256:d14eaa4d315e7688daa4969f616f226614350c48730bfa1692d2caebd8c90d40 \
     --hash=sha256:d3eab6605e3ec8b7a0863df09cc1d23714908fa61aa6986a845c20ba488b4efa
     # via cookiecutter
-jmespath==0.10.0 \
-    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
-    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f
+jmespath==1.0.1 \
+    --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \
+    --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe
     # via
     #   aws-sam-cli (setup.py)
     #   boto3
@@ -402,6 +402,10 @@ networkx==2.6.3 \
     --hash=sha256:80b6b89c77d1dfb64a4c7854981b60aeea6360ac02c6d4e4913319e0a313abef \
     --hash=sha256:c0946ed31d71f1b732b5aaa6da5a0388a345019af232ce2f49c766e2d6795c51
     # via cfn-lint
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via docker
 pbr==5.11.1 \
     --hash=sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b \
     --hash=sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3
@@ -703,7 +707,6 @@ six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
-    #   docker
     #   junit-xml
     #   python-dateutil
     #   serverlessrepo
@@ -742,6 +745,7 @@ urllib3==1.26.15 \
     --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42
     # via
     #   botocore
+    #   docker
     #   requests
 watchdog==2.1.2 \
     --hash=sha256:0237db4d9024859bea27d0efb59fe75eef290833fd988b8ead7a879b0308c2db \

samcli/cli/global_config.py

@@ -163,7 +163,7 @@ def get_value(
         self,
         config_entry: ConfigEntry,
         default: Optional[T] = None,
-        value_type: Type[T] = T,
+        value_type: Type[T] = T,  # type: ignore
         is_flag: bool = False,
         reload_config: bool = False,
     ) -> Optional[T]:

samcli/cli/hidden_imports.py

@@ -2,31 +2,22 @@
 Keeps list of hidden/dynamic imports that is being used in SAM CLI, so that pyinstaller can include these packages
 """
 import pkgutil
-from typing import cast
+from types import ModuleType
 
-from typing_extensions import Protocol
 
-
-class HasPathAndName(Protocol):
-    __path__: str
-    __name__: str
-
-
-def walk_modules(module: HasPathAndName, visited: set) -> None:
+def walk_modules(module: ModuleType, visited: set) -> None:
     """Recursively find all modules from a parent module"""
     for pkg in pkgutil.walk_packages(module.__path__, module.__name__ + "."):
         if pkg.name in visited:
             continue
         visited.add(pkg.name)
         if pkg.ispkg:
             submodule = __import__(pkg.name)
-            submodule = cast(HasPathAndName, submodule)
             walk_modules(submodule, visited)
 
 
 samcli_modules = set(["samcli"])
 samcli = __import__("samcli")
-samcli = cast(HasPathAndName, samcli)
 walk_modules(samcli, samcli_modules)
 
 SAM_CLI_HIDDEN_IMPORTS = list(samcli_modules) + [

samcli/commands/package/package_context.py

@@ -22,6 +22,7 @@
 import boto3
 import click
 import docker
+from docker.constants import DEFAULT_DOCKER_API_VERSION
 
 from samcli.commands.package.exceptions import PackageFailedError
 from samcli.lib.intrinsic_resolver.intrinsics_symbol_table import IntrinsicsSymbolTable
@@ -120,7 +121,7 @@ def run(self):
         )
         ecr_client = boto3.client("ecr", config=get_boto_config_with_user_agent(region_name=region_name))
 
-        docker_client = docker.from_env()
+        docker_client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
 
         s3_uploader = S3Uploader(
             s3_client, self.s3_bucket, self.s3_prefix, self.kms_key_id, self.force_upload, self.no_progressbar

samcli/hook_packages/terraform/hooks/prepare/resource_linking.py

@@ -243,7 +243,7 @@ def _link_using_terraform_config(self, source_tf_resource: TFResource, cfn_resou
             return
 
         for cfn_resource in cfn_resources:
-            self._resource_pair.cfn_resource_update_call_back_function(cfn_resource, dest_resources)  # type: ignore
+            self._resource_pair.cfn_resource_update_call_back_function(cfn_resource, dest_resources)
 
     def _link_using_linking_fields(self, cfn_resource: Dict) -> None:
         """
@@ -298,7 +298,7 @@ def _link_using_linking_fields(self, cfn_resource: Dict) -> None:
             return
 
         LOG.debug("The value of the source resource linking field after mapping %s", dest_resources)
-        self._resource_pair.cfn_resource_update_call_back_function(cfn_resource, dest_resources)  # type: ignore
+        self._resource_pair.cfn_resource_update_call_back_function(cfn_resource, dest_resources)
 
     def _process_resolved_resources(
         self,

samcli/hook_packages/terraform/lib/utils.py

@@ -74,7 +74,7 @@ def _calculate_configuration_attribute_value_hash(
     else:
         sorted_references_list = sorted(
             configuration_attribute_value,
-            key=lambda x: x.value if isinstance(x, ConstantValue) else f"{x.module_address}.{x.value}",  # type: ignore
+            key=lambda x: x.value if isinstance(x, ConstantValue) else f"{x.module_address}.{x.value}",
         )
         for ref in sorted_references_list:
             md5.update(

samcli/lib/build/app_builder.py

@@ -8,6 +8,7 @@
 import pathlib
 from typing import List, Optional, Dict, cast, NamedTuple
 import docker
+from docker.constants import DEFAULT_DOCKER_API_VERSION
 import docker.errors
 from aws_lambda_builders import (
     RPC_PROTOCOL_VERSION as lambda_builders_protocol_version,
@@ -156,7 +157,7 @@ def __init__(
         self._parallel = parallel
         self._mode = mode
         self._stream_writer = stream_writer if stream_writer else StreamWriter(stream=osutils.stderr(), auto_flush=True)
-        self._docker_client = docker_client if docker_client else docker.from_env()
+        self._docker_client = docker_client if docker_client else docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
 
         self._deprecated_runtimes = DEPRECATED_RUNTIMES
         self._colored = Colored()

samcli/lib/build/workflows.py

@@ -4,7 +4,7 @@
 from typing import List
 
 CONFIG = namedtuple(
-    "Capability",
+    "CONFIG",
     [
         "language",
         "dependency_manager",

samcli/lib/deploy/deployer.py

@@ -624,7 +624,7 @@ def sync(
             msg = ""
 
             if exists:
-                kwargs["DisableRollback"] = disable_rollback
+                kwargs["DisableRollback"] = disable_rollback  # type: ignore
 
                 result = self.update_stack(**kwargs)
                 self.wait_for_execute(stack_name, "UPDATE", disable_rollback, on_failure=on_failure)

samcli/lib/hook/hook_config.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import Dict, NamedTuple, Optional, cast
 
-import jsonschema  # type: ignore
+import jsonschema
 
 from .exceptions import InvalidHookPackageConfigException
 

samcli/lib/iac/cdk/cdk_iac.py

@@ -19,13 +19,13 @@ class CdkIacImplementation(IaCPluginInterface):
         the CDK project type
     """
 
-    def read_project(self, lookup_paths: List[LookupPath]) -> SamCliProject:
+    def read_project(self, lookup_paths: List[LookupPath]) -> SamCliProject:  # type: ignore
         pass
 
-    def write_project(self, project: SamCliProject, build_dir: str) -> bool:
+    def write_project(self, project: SamCliProject, build_dir: str) -> bool:  # type: ignore
         pass
 
-    def update_packaged_locations(self, stack: Stack) -> bool:
+    def update_packaged_locations(self, stack: Stack) -> bool:  # type: ignore
         pass
 
     @staticmethod

samcli/lib/iac/cfn/cfn_iac.py

@@ -1,7 +1,6 @@
 """
 Provide a CFN implementation of IaCPluginInterface
 """
-
 import logging
 import os
 from typing import List, Optional
@@ -72,11 +71,11 @@ def read_project(self, lookup_paths: List[LookupPath]) -> SamCliProject:
         stack = self._build_stack(self._template_file)
         return SamCliProject([stack])
 
-    def write_project(self, project: SamCliProject, build_dir: str) -> bool:
+    def write_project(self, project: SamCliProject, build_dir: str) -> bool:  # type: ignore
         # TODO
         pass
 
-    def update_packaged_locations(self, stack: Stack) -> bool:
+    def update_packaged_locations(self, stack: Stack) -> bool:  # type: ignore
         # TODO
         pass
 

samcli/lib/package/ecr_uploader.py

@@ -9,6 +9,7 @@
 import botocore
 import click
 import docker
+from docker.constants import DEFAULT_DOCKER_API_VERSION
 from docker.errors import APIError, BuildError
 
 from samcli.commands.package.exceptions import (
@@ -35,7 +36,7 @@ class ECRUploader:
     def __init__(
         self, docker_client, ecr_client, ecr_repo, ecr_repo_multi, no_progressbar=False, tag="latest", stream=stderr()
     ):
-        self.docker_client = docker_client if docker_client else docker.from_env()
+        self.docker_client = docker_client if docker_client else docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
         self.ecr_client = ecr_client
         self.ecr_repo = ecr_repo
         self.ecr_repo_multi = ecr_repo_multi

samcli/lib/package/image_utils.py

@@ -2,6 +2,7 @@
 Image artifacts based utilities
 """
 import docker
+from docker.constants import DEFAULT_DOCKER_API_VERSION
 from docker.errors import APIError, NullResource
 
 from samcli.commands.package.exceptions import DockerGetLocalImageFailedError
@@ -35,7 +36,7 @@ def tag_translation(image, docker_image_id=None, gen_tag="latest"):
 
     if not docker_image_id:
         try:
-            docker_client = docker.from_env()
+            docker_client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
             docker_image_id = docker_client.images.get(image).id
         except APIError as ex:
             raise DockerGetLocalImageFailedError(str(ex)) from ex