Merge branch 'rav/sync_response_cache' into rav/sw1v-hotfixes

.github/workflows/tests.yml

@@ -367,13 +367,16 @@ jobs:
       - name: Set build result
         env:
           NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines
+        # the `jq` incantation dumps out a series of "<job> <result>" lines.
+        # we set it to an intermediate variable to avoid a pipe, which makes it
+        # hard to set $rc.
         run: |
-          set -o pipefail
-          jq -r 'to_entries[] | [.key,.value.result] | join(" ")' \
-                          <<< $NEEDS_CONTEXT |
-              while read job result; do
-                  if [ "$result" != "success" ]; then
-                      echo "::set-failed ::Job $job returned $result"
-                  fi
-              done
+          rc=0
+          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
+          while read job result ; do
+              if [ "$result" != "success" ]; then
+                  echo "::set-failed ::Job $job returned $result"
+                  rc=1
+              fi
+          done <<< $results
+          exit $rc

CHANGES.md

@@ -1,3 +1,9 @@
+Synapse 1.39.0 (2021-07-29)
+===========================
+
+No significant changes.
+
+
 Synapse 1.39.0rc3 (2021-07-28)
 ==============================
 
@@ -19,10 +25,7 @@ Internal Changes
 Synapse 1.39.0rc2 (2021-07-22)
 ==============================
 
-Bugfixes
---------
-
-- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457))
+This release also includes the changes in v1.38.1.
 
 
 Internal Changes

changelog.d/10390.misc

@@ -0,0 +1 @@
+Prune inbound federation inbound queues for a room if they get too large.

changelog.d/10415.misc

@@ -0,0 +1 @@
+Remove shebang line from module files.

changelog.d/10439.bugfix

@@ -0,0 +1 @@
+Fix events with floating outlier state being rejected over federation.

changelog.d/10440.feature

@@ -0,0 +1 @@
+Allow setting transaction limit for database connections.

changelog.d/10447.feature

@@ -0,0 +1 @@
+Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events.

changelog.d/10491.misc

@@ -0,0 +1 @@
+Improve type annotations for `ObservableDeferred`.

changelog.d/10499.bugfix

@@ -0,0 +1 @@
+Fix a bug which caused an explicit assignment of power-level 0 to a user to be misinterpreted in rare circumstances.

changelog.d/10500.misc

@@ -0,0 +1 @@
+Fix a bug which caused production debian packages to be incorrectly marked as 'prerelease'.

changelog.d/10511.feature

@@ -0,0 +1 @@
+Allow setting transaction limit for database connections.

changelog.d/10512.misc

@@ -0,0 +1 @@
+Update the `tests-done` Github Actions status.

changelog.d/10513.feature

@@ -0,0 +1 @@
+Add a configuration setting for the time a `/sync` response is cached for.

debian/changelog

@@ -4,6 +4,12 @@ matrix-synapse-py3 (1.39.0ubuntu1) UNRELEASED; urgency=medium
 
  -- Richard van der Hoff <richard@matrix.org>  Tue, 20 Jul 2021 00:10:03 +0100
 
+matrix-synapse-py3 (1.39.0) stable; urgency=medium
+
+  * New synapse release 1.39.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 29 Jul 2021 09:59:00 +0100
+
 matrix-synapse-py3 (1.39.0~rc3) stable; urgency=medium
 
   * New synapse release 1.39.0~rc3.

docker/build_debian.sh

@@ -11,10 +11,6 @@ DIST=`cut -d ':' -f2 <<< $distro`
 cp -aT /synapse/source /synapse/build
 cd /synapse/build
 
-# add an entry to the changelog for this distribution
-dch -M -l "+$DIST" "build for $DIST"
-dch -M -r "" --force-distribution --distribution "$DIST"
-
 # if this is a prerelease, set the Section accordingly.
 #
 # When the package is later added to the package repo, reprepro will use the
@@ -23,11 +19,14 @@ dch -M -r "" --force-distribution --distribution "$DIST"
 
 DEB_VERSION=`dpkg-parsechangelog -SVersion`
 case $DEB_VERSION in
-    *rc*|*a*|*b*|*c*)
+    *~rc*|*~a*|*~b*|*~c*)
         sed -ie '/^Section:/c\Section: prerelease' debian/control
         ;;
 esac
 
+# add an entry to the changelog for this distribution
+dch -M -l "+$DIST" "build for $DIST"
+dch -M -r "" --force-distribution --distribution "$DIST"
 
 dpkg-buildpackage -us -uc
 

docs/sample_config.yaml

@@ -711,6 +711,15 @@ caches:
   #
   #expiry_time: 30m
 
+  # Controls how long the results of a /sync request are cached for after
+  # a successful response is returned. A higher duration can help clients with
+  # intermittent connections, at the cost of higher memory usage.
+  #
+  # By default, this is zero, which means that sync responses are not cached
+  # at all.
+  #
+  #sync_response_cache_duration: 2m
+
 
 ## Database ##
 
@@ -720,6 +729,9 @@ caches:
 # 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
 # 'psycopg2' (for PostgreSQL).
 #
+# 'txn_limit' gives the maximum number of transactions to run per connection
+# before reconnecting. Defaults to 0, which means no limit.
+#
 # 'args' gives options which are passed through to the database engine,
 # except for options starting 'cp_', which are used to configure the Twisted
 # connection pool. For a reference to valid arguments, see:
@@ -740,6 +752,7 @@ caches:
 #
 #database:
 #  name: psycopg2
+#  txn_limit: 10000
 #  args:
 #    user: synapse_user
 #    password: secretpassword

synapse/__init__.py

@@ -47,7 +47,7 @@
 except ImportError:
     pass
 
-__version__ = "1.39.0rc3"
+__version__ = "1.39.0"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/_scripts/review_recent_signups.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/admin_cmd.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/appservice.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/client_reader.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/event_creator.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/federation_reader.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/federation_sender.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/frontend_proxy.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/generic_worker.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #

synapse/app/homeserver.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #

synapse/app/media_repository.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/pusher.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/synchrotron.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/user_dir.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/config/cache.py

@@ -151,6 +151,15 @@ def generate_config_section(self, **kwargs):
           # entries are never evicted based on time.
           #
           #expiry_time: 30m
+
+          # Controls how long the results of a /sync request are cached for after
+          # a successful response is returned. A higher duration can help clients with
+          # intermittent connections, at the cost of higher memory usage.
+          #
+          # By default, this is zero, which means that sync responses are not cached
+          # at all.
+          #
+          #sync_response_cache_duration: 2m
         """
 
     def read_config(self, config, **kwargs):
@@ -212,6 +221,10 @@ def read_config(self, config, **kwargs):
         else:
             self.expiry_time_msec = None
 
+        self.sync_response_cache_duration = self.parse_duration(
+            cache_config.get("sync_response_cache_duration", 0)
+        )
+
         # Resize all caches (if necessary) with the new factors we've loaded
         self.resize_all_caches()
 

synapse/config/database.py

@@ -33,6 +33,9 @@
 # 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
 # 'psycopg2' (for PostgreSQL).
 #
+# 'txn_limit' gives the maximum number of transactions to run per connection
+# before reconnecting. Defaults to 0, which means no limit.
+#
 # 'args' gives options which are passed through to the database engine,
 # except for options starting 'cp_', which are used to configure the Twisted
 # connection pool. For a reference to valid arguments, see:
@@ -53,6 +56,7 @@
 #
 #database:
 #  name: psycopg2
+#  txn_limit: 10000
 #  args:
 #    user: synapse_user
 #    password: secretpassword

synapse/event_auth.py

@@ -692,7 +692,7 @@ def get_user_power_level(user_id: str, auth_events: StateMap[EventBase]) -> int:
     power_level_event = get_power_level_event(auth_events)
     if power_level_event:
         level = power_level_event.content.get("users", {}).get(user_id)
-        if not level:
+        if level is None:
             level = power_level_event.content.get("users_default", 0)
 
         if level is None:

synapse/federation/federation_client.py

@@ -22,6 +22,7 @@
     Awaitable,
     Callable,
     Collection,
+    Container,
     Dict,
     Iterable,
     List,
@@ -513,6 +514,7 @@ async def _try_destination_list(
         description: str,
         destinations: Iterable[str],
         callback: Callable[[str], Awaitable[T]],
+        failover_errcodes: Optional[Container[str]] = None,
         failover_on_unknown_endpoint: bool = False,
     ) -> T:
         """Try an operation on a series of servers, until it succeeds
@@ -533,6 +535,9 @@ async def _try_destination_list(
                 next server tried. Normally the stacktrace is logged but this is
                 suppressed if the exception is an InvalidResponseError.
 
+            failover_errcodes: Error codes (specific to this endpoint) which should
+                cause a failover when received as part of an HTTP 400 error.
+
             failover_on_unknown_endpoint: if True, we will try other servers if it looks
                 like a server doesn't support the endpoint. This is typically useful
                 if the endpoint in question is new or experimental.
@@ -544,6 +549,9 @@ async def _try_destination_list(
             SynapseError if the chosen remote server returns a 300/400 code, or
             no servers were reachable.
         """
+        if failover_errcodes is None:
+            failover_errcodes = ()
+
         for destination in destinations:
             if destination == self.server_name:
                 continue
@@ -558,11 +566,17 @@ async def _try_destination_list(
                 synapse_error = e.to_synapse_error()
                 failover = False
 
-                # Failover on an internal server error, or if the destination
-                # doesn't implemented the endpoint for some reason.
+                # Failover should occur:
+                #
+                # * On internal server errors.
+                # * If the destination responds that it cannot complete the request.
+                # * If the destination doesn't implemented the endpoint for some reason.
                 if 500 <= e.code < 600:
                     failover = True
 
+                elif e.code == 400 and synapse_error.errcode in failover_errcodes:
+                    failover = True
+
                 elif failover_on_unknown_endpoint and self._is_unknown_endpoint(
                     e, synapse_error
                 ):
@@ -678,8 +692,20 @@ async def send_request(destination: str) -> Tuple[str, EventBase, RoomVersion]:
 
             return destination, ev, room_version
 
+        # MSC3083 defines additional error codes for room joins. Unfortunately
+        # we do not yet know the room version, assume these will only be returned
+        # by valid room versions.
+        failover_errcodes = (
+            (Codes.UNABLE_AUTHORISE_JOIN, Codes.UNABLE_TO_GRANT_JOIN)
+            if membership == Membership.JOIN
+            else None
+        )
+
         return await self._try_destination_list(
-            "make_" + membership, destinations, send_request
+            "make_" + membership,
+            destinations,
+            send_request,
+            failover_errcodes=failover_errcodes,
         )
 
     async def send_join(
@@ -818,7 +844,14 @@ async def _execute(pdu: EventBase) -> None:
                 origin=destination,
             )
 
+        # MSC3083 defines additional error codes for room joins.
+        failover_errcodes = None
         if room_version.msc3083_join_rules:
+            failover_errcodes = (
+                Codes.UNABLE_AUTHORISE_JOIN,
+                Codes.UNABLE_TO_GRANT_JOIN,
+            )
+
             # If the join is being authorised via allow rules, we need to send
             # the /send_join back to the same server that was originally used
             # with /make_join.
@@ -827,7 +860,9 @@ async def _execute(pdu: EventBase) -> None:
                     get_domain_from_id(pdu.content["join_authorised_via_users_server"])
                 ]
 
-        return await self._try_destination_list("send_join", destinations, send_request)
+        return await self._try_destination_list(
+            "send_join", destinations, send_request, failover_errcodes=failover_errcodes
+        )
 
     async def _do_send_join(
         self, room_version: RoomVersion, destination: str, pdu: EventBase