Merge branch 'master' into fix_docs

CHANGES/7954.feature

@@ -0,0 +1 @@
+Implement happy eyeballs

CHANGES/8012.bugfix

@@ -0,0 +1 @@
+Fix :py:class:`~aiohttp.web.FileResponse`. doing blocking I/O in the event loop

CHANGES/8014.bugfix

@@ -0,0 +1 @@
+Fix double compress when compression enabled and compressed file exists

CHANGES/8021.bugfix

@@ -0,0 +1 @@
+Add runtime type check for ``ClientSession`` ``timeout`` parameter.

CONTRIBUTORS.txt

@@ -149,6 +149,7 @@ Hugo Hromic
 Hugo van Kemenade
 Hynek Schlawack
 Igor Alexandrov
+Igor Bolshakov
 Igor Davydenko
 Igor Mozharovsky
 Igor Pavlov

aiohttp/client.py

@@ -269,9 +269,13 @@ def __init__(
         self._version = version
         self._json_serialize = json_serialize
         if timeout is sentinel or timeout is None:
-            self._timeout = DEFAULT_TIMEOUT
-        else:
-            self._timeout = timeout
+            timeout = DEFAULT_TIMEOUT
+        if not isinstance(timeout, ClientTimeout):
+            raise ValueError(
+                f"timeout parameter cannot be of {type(timeout)} type, "
+                "please use 'timeout=ClientTimeout(...)'",
+            )
+        self._timeout = timeout
         self._raise_for_status = raise_for_status
         self._auto_decompress = auto_decompress
         self._trust_env = trust_env

aiohttp/connector.py

@@ -3,6 +3,7 @@
 import functools
 import logging
 import random
+import socket
 import sys
 import traceback
 import warnings
@@ -31,6 +32,8 @@
     cast,
 )
 
+import aiohappyeyeballs
+
 from . import hdrs, helpers
 from .abc import AbstractResolver
 from .client_exceptions import (
@@ -730,6 +733,10 @@ class TCPConnector(BaseConnector):
     limit_per_host - Number of simultaneous connections to one host.
     enable_cleanup_closed - Enables clean-up closed ssl transports.
                             Disabled by default.
+    happy_eyeballs_delay - This is the “Connection Attempt Delay”
+                           as defined in RFC 8305. To disable
+                           the happy eyeballs algorithm, set to None.
+    interleave - “First Address Family Count” as defined in RFC 8305
     loop - Optional event loop.
     """
 
@@ -748,6 +755,8 @@ def __init__(
         limit_per_host: int = 0,
         enable_cleanup_closed: bool = False,
         timeout_ceil_threshold: float = 5,
+        happy_eyeballs_delay: Optional[float] = 0.25,
+        interleave: Optional[int] = None,
     ) -> None:
         super().__init__(
             keepalive_timeout=keepalive_timeout,
@@ -772,7 +781,9 @@ def __init__(
         self._cached_hosts = _DNSCacheTable(ttl=ttl_dns_cache)
         self._throttle_dns_events: Dict[Tuple[str, int], EventResultOrError] = {}
         self._family = family
-        self._local_addr = local_addr
+        self._local_addr_infos = aiohappyeyeballs.addr_to_addr_infos(local_addr)
+        self._happy_eyeballs_delay = happy_eyeballs_delay
+        self._interleave = interleave
 
     def _close_immediately(self) -> List["asyncio.Future[None]"]:
         for ev in self._throttle_dns_events.values():
@@ -956,6 +967,7 @@ def _get_fingerprint(self, req: ClientRequest) -> Optional["Fingerprint"]:
     async def _wrap_create_connection(
         self,
         *args: Any,
+        addr_infos: List[aiohappyeyeballs.AddrInfoType],
         req: ClientRequest,
         timeout: "ClientTimeout",
         client_error: Type[Exception] = ClientConnectorError,
@@ -965,7 +977,14 @@ async def _wrap_create_connection(
             async with ceil_timeout(
                 timeout.sock_connect, ceil_threshold=timeout.ceil_threshold
             ):
-                return await self._loop.create_connection(*args, **kwargs)
+                sock = await aiohappyeyeballs.start_connection(
+                    addr_infos=addr_infos,
+                    local_addr_infos=self._local_addr_infos,
+                    happy_eyeballs_delay=self._happy_eyeballs_delay,
+                    interleave=self._interleave,
+                    loop=self._loop,
+                )
+                return await self._loop.create_connection(*args, **kwargs, sock=sock)
         except cert_errors as exc:
             raise ClientConnectorCertificateError(req.connection_key, exc) from exc
         except ssl_errors as exc:
@@ -1076,6 +1095,27 @@ async def _start_tls_connection(
 
         return tls_transport, tls_proto
 
+    def _convert_hosts_to_addr_infos(
+        self, hosts: List[Dict[str, Any]]
+    ) -> List[aiohappyeyeballs.AddrInfoType]:
+        """Converts the list of hosts to a list of addr_infos.
+
+        The list of hosts is the result of a DNS lookup. The list of
+        addr_infos is the result of a call to `socket.getaddrinfo()`.
+        """
+        addr_infos: List[aiohappyeyeballs.AddrInfoType] = []
+        for hinfo in hosts:
+            host = hinfo["host"]
+            is_ipv6 = ":" in host
+            family = socket.AF_INET6 if is_ipv6 else socket.AF_INET
+            if self._family and self._family != family:
+                continue
+            addr = (host, hinfo["port"], 0, 0) if is_ipv6 else (host, hinfo["port"])
+            addr_infos.append(
+                (family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr)
+            )
+        return addr_infos
+
     async def _create_direct_connection(
         self,
         req: ClientRequest,
@@ -1120,36 +1160,27 @@ def drop_exception(fut: "asyncio.Future[List[Dict[str, Any]]]") -> None:
             raise ClientConnectorError(req.connection_key, exc) from exc
 
         last_exc: Optional[Exception] = None
-
-        for hinfo in hosts:
-            host = hinfo["host"]
-            port = hinfo["port"]
-
+        addr_infos = self._convert_hosts_to_addr_infos(hosts)
+        while addr_infos:
             # Strip trailing dots, certificates contain FQDN without dots.
             # See https://github.com/aio-libs/aiohttp/issues/3636
             server_hostname = (
-                (req.server_hostname or hinfo["hostname"]).rstrip(".")
-                if sslcontext
-                else None
+                (req.server_hostname or host).rstrip(".") if sslcontext else None
             )
 
             try:
                 transp, proto = await self._wrap_create_connection(
                     self._factory,
-                    host,
-                    port,
                     timeout=timeout,
                     ssl=sslcontext,
-                    family=hinfo["family"],
-                    proto=hinfo["proto"],
-                    flags=hinfo["flags"],
+                    addr_infos=addr_infos,
                     server_hostname=server_hostname,
-                    local_addr=self._local_addr,
                     req=req,
                     client_error=client_error,
                 )
             except ClientConnectorError as exc:
                 last_exc = exc
+                aiohappyeyeballs.pop_addr_infos_interleave(addr_infos, self._interleave)
                 continue
 
             if req.is_ssl() and fingerprint:
@@ -1160,6 +1191,10 @@ def drop_exception(fut: "asyncio.Future[List[Dict[str, Any]]]") -> None:
                     if not self._cleanup_closed_disabled:
                         self._cleanup_closed_transports.append(transp)
                     last_exc = exc
+                    # Remove the bad peer from the list of addr_infos
+                    sock: socket.socket = transp.get_extra_info("socket")
+                    bad_peer = sock.getpeername()
+                    aiohappyeyeballs.remove_addr_infos(addr_infos, bad_peer)
                     continue
 
             return transp, proto

aiohttp/web_fileresponse.py

@@ -121,19 +121,31 @@ async def _precondition_failed(
         self.content_length = 0
         return await super().prepare(request)
 
-    async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]:
+    def _get_file_path_stat_and_gzip(
+        self, check_for_gzipped_file: bool
+    ) -> Tuple[pathlib.Path, os.stat_result, bool]:
+        """Return the file path, stat result, and gzip status.
+
+        This method should be called from a thread executor
+        since it calls os.stat which may block.
+        """
         filepath = self._path
-
-        gzip = False
-        if "gzip" in request.headers.get(hdrs.ACCEPT_ENCODING, ""):
+        if check_for_gzipped_file:
             gzip_path = filepath.with_name(filepath.name + ".gz")
+            try:
+                return gzip_path, gzip_path.stat(), True
+            except OSError:
+                # Fall through and try the non-gzipped file
+                pass
 
-            if gzip_path.is_file():
-                filepath = gzip_path
-                gzip = True
+        return filepath, filepath.stat(), False
 
+    async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]:
         loop = asyncio.get_event_loop()
-        st: os.stat_result = await loop.run_in_executor(None, filepath.stat)
+        check_for_gzipped_file = "gzip" in request.headers.get(hdrs.ACCEPT_ENCODING, "")
+        filepath, st, gzip = await loop.run_in_executor(
+            None, self._get_file_path_stat_and_gzip, check_for_gzipped_file
+        )
 
         etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}"
         last_modified = st.st_mtime
@@ -251,6 +263,10 @@ async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter
             self.headers[hdrs.CONTENT_ENCODING] = encoding
         if gzip:
             self.headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING
+            # Disable compression if we are already sending
+            # a compressed file since we don't want to double
+            # compress.
+            self._compression = False
 
         self.etag = etag_value  # type: ignore[assignment]
         self.last_modified = st.st_mtime  # type: ignore[assignment]

docs/client_reference.rst

@@ -1055,7 +1055,8 @@ is controlled by *force_close* constructor's parameter).
                  family=0, ssl_context=None, local_addr=None, \
                  resolver=None, keepalive_timeout=sentinel, \
                  force_close=False, limit=100, limit_per_host=0, \
-                 enable_cleanup_closed=False, loop=None)
+                 enable_cleanup_closed=False, timeout_ceil_threshold=5, \
+                 happy_eyeballs_delay=0.25, interleave=None, loop=None)
 
    Connector for working with *HTTP* and *HTTPS* via *TCP* sockets.
 
@@ -1158,6 +1159,24 @@ is controlled by *force_close* constructor's parameter).
       If this parameter is set to True, aiohttp additionally aborts underlining
       transport after 2 seconds. It is off by default.
 
+   :param float happy_eyeballs_delay: The amount of time in seconds to wait for a
+      connection attempt to complete, before starting the next attempt in parallel.
+      This is the “Connection Attempt Delay” as defined in RFC 8305. To disable
+      Happy Eyeballs, set this to ``None``. The default value recommended by the
+      RFC is 0.25 (250 milliseconds).
+
+        .. versionadded:: 3.10
+
+   :param int interleave: controls address reordering when a host name resolves
+      to multiple IP addresses. If ``0`` or unspecified, no reordering is done, and
+      addresses are tried in the order returned by the resolver. If a positive
+      integer is specified, the addresses are interleaved by address family, and
+      the given integer is interpreted as “First Address Family Count” as defined
+      in RFC 8305. The default is ``0`` if happy_eyeballs_delay is not specified, and
+      ``1`` if it is.
+
+        .. versionadded:: 3.10
+
    .. attribute:: family
 
       *TCP* socket family e.g. :data:`socket.AF_INET` or

requirements/base.txt

@@ -6,6 +6,8 @@
 #
 aiodns==3.1.1 ; sys_platform == "linux" or sys_platform == "darwin"
     # via -r requirements/runtime-deps.in
+aiohappyeyeballs==2.3.2
+    # via -r requirements/runtime-deps.in
 aiosignal==1.3.1
     # via -r requirements/runtime-deps.in
 async-timeout==4.0.3 ; python_version < "3.11"
@@ -14,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython"
     # via -r requirements/runtime-deps.in
 cffi==1.15.1
     # via pycares
-frozenlist==1.4.0
+frozenlist==1.4.1
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal

requirements/constraints.txt

@@ -6,6 +6,8 @@
 #
 aiodns==3.1.1 ; sys_platform == "linux" or sys_platform == "darwin"
     # via -r requirements/runtime-deps.in
+aiohappyeyeballs==2.3.2
+    # via -r requirements/runtime-deps.in
 aiohttp-theme==0.1.6
     # via -r requirements/doc.in
 aioredis==2.0.1
@@ -46,15 +48,15 @@ click==8.1.6
     #   towncrier
     #   typer
     #   wait-for-it
-coverage==7.3.2
+coverage==7.4.0
     # via
     #   -r requirements/test.in
     #   pytest-cov
 cryptography==41.0.3
     # via
     #   pyjwt
     #   trustme
-cython==3.0.7
+cython==3.0.8
     # via -r requirements/cython.in
 distlib==0.3.7
     # via virtualenv
@@ -64,9 +66,9 @@ exceptiongroup==1.1.2
     # via pytest
 filelock==3.12.2
     # via virtualenv
-freezegun==1.3.1
+freezegun==1.4.0
     # via -r requirements/test.in
-frozenlist==1.4.0
+frozenlist==1.4.1
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -148,7 +150,7 @@ pyjwt==2.8.0
     #   pyjwt
 pyproject-hooks==1.0.0
     # via build
-pytest==7.4.3
+pytest==7.4.4
     # via
     #   -r requirements/lint.in
     #   -r requirements/test.in

requirements/cython.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --allow-unsafe --output-file=requirements/cython.txt --resolver=backtracking --strip-extras requirements/cython.in
 #
-cython==3.0.7
+cython==3.0.8
     # via -r requirements/cython.in
 multidict==6.0.4
     # via -r requirements/multidict.in

requirements/dev.txt

@@ -6,6 +6,8 @@
 #
 aiodns==3.1.1 ; sys_platform == "linux" or sys_platform == "darwin"
     # via -r requirements/runtime-deps.in
+aiohappyeyeballs==2.3.2
+    # via -r requirements/runtime-deps.in
 aiohttp-theme==0.1.6
     # via -r requirements/doc.in
 aioredis==2.0.1
@@ -46,7 +48,7 @@ click==8.1.6
     #   towncrier
     #   typer
     #   wait-for-it
-coverage==7.3.2
+coverage==7.4.0
     # via
     #   -r requirements/test.in
     #   pytest-cov
@@ -62,9 +64,9 @@ exceptiongroup==1.1.2
     # via pytest
 filelock==3.12.2
     # via virtualenv
-freezegun==1.3.1
+freezegun==1.4.0
     # via -r requirements/test.in
-frozenlist==1.4.0
+frozenlist==1.4.1
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -143,7 +145,7 @@ pyjwt==2.8.0
     #   pyjwt
 pyproject-hooks==1.0.0
     # via build
-pytest==7.4.3
+pytest==7.4.4
     # via
     #   -r requirements/lint.in
     #   -r requirements/test.in