Update main branch to current v0.6.5 pre-release state

examples/encryption_appservice.ts

@@ -36,7 +36,7 @@ const worksImage = fs.readFileSync("./examples/static/it-works.png");
 const registration: IAppserviceRegistration = {
     "as_token": creds?.['asToken'] ?? "change_me",
     "hs_token": creds?.['hsToken'] ?? "change_me",
-    "sender_localpart": "crypto_main_bot_user",
+    "sender_localpart": "crypto_main_bot_user2",
     "namespaces": {
         users: [{
             regex: "@crypto.*:localhost",
@@ -95,6 +95,16 @@ const bot = appservice.botIntent;
         });
     }
 
+    appservice.on("query.key_claim", (req, done) => {
+        LogService.info("index", "Key claim request:", req);
+        done({});
+    });
+
+    appservice.on("query.key", (req, done) => {
+        LogService.info("index", "Key query request:", req);
+        done({});
+    });
+
     appservice.on("room.failed_decryption", async (roomId: string, event: any, e: Error) => {
         LogService.error("index", `Failed to decrypt ${roomId} ${event['event_id']} because `, e);
     });

package.json

@@ -51,7 +51,7 @@
     "tsconfig.json"
   ],
   "dependencies": {
-    "@matrix-org/matrix-sdk-crypto-nodejs": "^0.1.0-beta.3",
+    "@matrix-org/matrix-sdk-crypto-nodejs": "0.1.0-beta.3",
     "@types/express": "^4.17.13",
     "another-json": "^0.2.0",
     "async-lock": "^1.3.2",
@@ -67,7 +67,7 @@
     "morgan": "^1.10.0",
     "request": "^2.88.2",
     "request-promise": "^4.2.6",
-    "sanitize-html": "^2.7.0"
+    "sanitize-html": "^2.8.0"
   },
   "devDependencies": {
     "@babel/core": "^7.18.2",

src/MatrixClient.ts

@@ -676,7 +676,7 @@ export class MatrixClient extends EventEmitter {
 
         if (createFilter && filter) {
             LogService.trace("MatrixClientLite", "Creating new filter");
-            return this.doRequest("POST", "/_matrix/client/v3/user/" + encodeURIComponent(userId) + "/filter", null, filter).then(async response => {
+            await this.doRequest("POST", "/_matrix/client/v3/user/" + encodeURIComponent(userId) + "/filter", null, filter).then(async response => {
                 this.filterId = response["filter_id"];
                 // noinspection ES6RedundantAwait
                 await Promise.resolve(this.storage.setSyncToken(null));
@@ -1162,11 +1162,12 @@ export class MatrixClient extends EventEmitter {
     /**
      * Leaves the given room
      * @param {string} roomId the room ID to leave
+     * @param {string=} reason Optional reason to be included as the reason for leaving the room.
      * @returns {Promise<any>} resolves when left
      */
     @timedMatrixClientFunctionCall()
-    public leaveRoom(roomId: string): Promise<any> {
-        return this.doRequest("POST", "/_matrix/client/v3/rooms/" + encodeURIComponent(roomId) + "/leave", null, {});
+    public leaveRoom(roomId: string, reason?: string): Promise<any> {
+        return this.doRequest("POST", "/_matrix/client/v3/rooms/" + encodeURIComponent(roomId) + "/leave", null, { reason });
     }
 
     /**

src/UnstableApis.ts

@@ -28,7 +28,7 @@ export class UnstableApis {
      * @returns {Promise<string>} Resolves to the event ID of the reaction
      */
     public async addReactionToEvent(roomId: string, eventId: string, emoji: string): Promise<string> {
-        return this.client.sendEvent(roomId, "m.reaction", {
+        return this.client.sendRawEvent(roomId, "m.reaction", {
             "m.relates_to": {
                 event_id: eventId,
                 key: emoji,

src/appservice/Intent.ts

@@ -169,7 +169,7 @@ export class Intent {
                     }
 
                     // Now set up crypto
-                    await this.client.crypto.prepare(await this.client.getJoinedRooms());
+                    await this.client.crypto.prepare(await this.refreshJoinedRooms());
 
                     this.appservice.on("room.event", (roomId, event) => {
                         if (!this.knownJoinedRooms.includes(roomId)) return;
@@ -201,12 +201,13 @@ export class Intent {
     /**
      * Leaves the given room.
      * @param {string} roomId The room ID to leave
+     * @param {string=} reason Optional reason to be included as the reason for leaving the room.
      * @returns {Promise<any>} Resolves when the room has been left.
      */
     @timedIntentFunctionCall()
-    public async leaveRoom(roomId: string): Promise<any> {
+    public async leaveRoom(roomId: string, reason?: string): Promise<any> {
         await this.ensureRegistered();
-        return this.client.leaveRoom(roomId).then(async () => {
+        return this.client.leaveRoom(roomId, reason).then(async () => {
             // Recalculate joined rooms now that we've left a room
             await this.refreshJoinedRooms();
         });

src/appservice/http_responses.ts

@@ -22,3 +22,56 @@ interface IProtocolInstance {
     fields: { [field: string]: string };
     network_id: string;
 }
+
+/**
+ * This is the response format for an MSC3983 `/keys/claim` request.
+ * See https://github.com/matrix-org/matrix-spec-proposals/pull/3983
+ * @deprecated This can be removed at any time without notice as it is unstable functionality.
+ * @category Application services
+ */
+export interface MSC3983KeyClaimResponse {
+    [userId: string]: {
+        [deviceId: string]: {
+            [keyId: string]: {
+                // for signed_curve25519 keys
+                key: string;
+                signatures: {
+                    [userId: string]: {
+                        [keyId: string]: string;
+                    };
+                };
+            };
+        };
+    };
+}
+
+/**
+ * This is the response format for an MSC3984 `/keys/query` request.
+ * See https://github.com/matrix-org/matrix-spec-proposals/pull/3984
+ * @deprecated This can be removed at any time without notice as it is unstable functionality.
+ * @category Application services
+ */
+export interface MSC3984KeyQueryResponse {
+    device_keys: {
+        [userId: string]: {
+            [deviceId: string]: {
+                algorithms: string[];
+                device_id: string;
+                user_id: string;
+                keys: {
+                    [keyId: string]: string;
+                };
+                signatures: {
+                    [userId: string]: {
+                        [keyId: string]: string;
+                    };
+                };
+                unsigned?: {
+                    [key: string]: any;
+                };
+            };
+        };
+    };
+
+    // TODO: Cross-signing support
+}

src/e2ee/RustEngine.ts

@@ -34,7 +34,11 @@ export class RustEngine {
     public constructor(public readonly machine: OlmMachine, private client: MatrixClient) {
     }
 
-    public async run(...types: RequestType[]) {
+    public async run() {
+        await this.runOnly(); // run everything, but with syntactic sugar
+    }
+
+    private async runOnly(...types: RequestType[]) {
         // Note: we should not be running this until it runs out, so cache the value into a variable
         const requests = await this.machine.outgoingRequests();
         for (const request of requests) {
@@ -122,9 +126,10 @@ export class RustEngine {
         settings.rotationPeriod = BigInt(encEv.rotationPeriodMs);
         settings.rotationPeriodMessages = BigInt(encEv.rotationPeriodMessages);
 
-        await this.run(RequestType.KeysQuery);
         await this.lock.acquire(SYNC_LOCK_NAME, async () => {
-            const keysClaim = await this.machine.getMissingSessions(membersArray);
+            await this.machine.updateTrackedUsers(members); // just in case we missed some
+            await this.runOnly(RequestType.KeysQuery);
+            const keysClaim = await this.machine.getMissingSessions(members);
             if (keysClaim) {
                 await this.processKeysClaimRequest(keysClaim);
             }
@@ -144,7 +149,9 @@ export class RustEngine {
     }
 
     private async processKeysUploadRequest(request: KeysUploadRequest) {
-        const resp = await this.client.doRequest("POST", "/_matrix/client/v3/keys/upload", null, JSON.parse(request.body));
+        const body = JSON.parse(request.body);
+        // delete body["one_time_keys"]; // use this to test MSC3983
+        const resp = await this.client.doRequest("POST", "/_matrix/client/v3/keys/upload", null, body);
         await this.machine.markRequestAsSent(request.id, request.type, JSON.stringify(resp));
     }
 
@@ -155,7 +162,7 @@ export class RustEngine {
 
     private async processToDeviceRequest(request: ToDeviceRequest) {
         const req = JSON.parse(request.body);
-        await this.actuallyProcessToDeviceRequest(req.id, req.event_type, req.messages);
+        await this.actuallyProcessToDeviceRequest(req.txn_id, req.event_type, req.messages);
     }
 
     private async actuallyProcessToDeviceRequest(id: string, type: string, messages: Record<string, Record<string, unknown>>) {

src/helpers/Permalinks.ts

@@ -5,22 +5,22 @@
  */
 export interface PermalinkParts {
     /**
-     * The room ID or alias the permalink references. May be null.
+     * The room ID or alias the permalink references. May be undefined.
      */
     roomIdOrAlias: string;
 
     /**
-     * The user ID the permalink references. May be null.
+     * The user ID the permalink references. May be undefined.
      */
     userId: string;
 
     /**
-     * The event ID the permalink references. May be null.
+     * The event ID the permalink references. May be undefined.
      */
     eventId: string;
 
     /**
-     * The servers the permalink is routed through. May be null or empty.
+     * The servers the permalink is routed through. May be undefined or empty.
      */
     viaServers: string[];
 }
@@ -38,7 +38,7 @@ export class Permalinks {
     private static encodeViaArgs(servers: string[]): string {
         if (!servers || !servers.length) return "";
 
-        return `?via=${servers.join("via=")}`;
+        return `?via=${servers.join("&via=")}`;
     }
 
     /**
@@ -77,32 +77,25 @@ export class Permalinks {
      * @returns {PermalinkParts} The parts of the permalink.
      */
     public static parseUrl(matrixTo: string): PermalinkParts {
-        if (!matrixTo || !matrixTo.startsWith("https://matrix.to/#/")) {
+        const matrixToRegexp = /^https:\/\/matrix\.to\/#\/(?<entity>[^/?]+)\/?(?<eventId>[^?]+)?(?<query>\?[^]*)?$/;
+
+        const url = matrixToRegexp.exec(matrixTo)?.groups;
+        if (!url) {
             throw new Error("Not a valid matrix.to URL");
         }
 
-        const parts = matrixTo.substring("https://matrix.to/#/".length).split("/");
-
-        const entity = decodeURIComponent(parts[0]);
+        const entity = decodeURIComponent(url.entity);
         if (entity[0] === '@') {
             return { userId: entity, roomIdOrAlias: undefined, eventId: undefined, viaServers: undefined };
         } else if (entity[0] === '#' || entity[0] === '!') {
-            if (parts.length === 1) {
-                return { roomIdOrAlias: entity, userId: undefined, eventId: undefined, viaServers: [] };
-            }
-
-            // rejoin the rest because v3 room can have slashes
-            const eventIdAndQuery = decodeURIComponent(parts.length > 1 ? parts.slice(1).join('/') : "");
-            const secondaryParts = eventIdAndQuery.split('?');
-
-            const eventId = secondaryParts[0];
-            const query = secondaryParts.length > 1 ? secondaryParts[1] : "";
-
-            const via = query.split("via=").filter(p => !!p);
-
-            return { roomIdOrAlias: entity, eventId: eventId, viaServers: via, userId: undefined };
+            return {
+                userId: undefined,
+                roomIdOrAlias: entity,
+                eventId: url.eventId && decodeURIComponent(url.eventId),
+                viaServers: new URLSearchParams(url.query).getAll('via'),
+            };
+        } else {
+            throw new Error("Unexpected entity");
         }
-
-        throw new Error("Unexpected entity");
     }
 }

test/MatrixClientTest.ts

@@ -1177,10 +1177,16 @@ describe('MatrixClient', () => {
             http.when("POST", "/_matrix/client/v3/user").respond(200, (path, content) => {
                 expect(path).toEqual(`${hsUrl}/_matrix/client/v3/user/${encodeURIComponent(userId)}/filter`);
                 expect(content).toMatchObject(filter);
-                client.stop(); // avoid a sync early
                 return { filter_id: filterId };
             });
 
+            // noinspection TypeScriptValidateJSTypes
+            http.when("GET", "/_matrix/client/v3/sync").respond(200, (path, content, req) => {
+                expect(req.queryParams.filter).toBe(filterId);
+                client.stop();
+                return { next_batch: "123" };
+            });
+
             await Promise.all([client.start(filter), http.flushAllExpected()]);
             expect(setFilterFn.callCount).toBe(1);
             expect(dmsMock.callCount).toBe(1);
@@ -1215,10 +1221,16 @@ describe('MatrixClient', () => {
             http.when("POST", "/_matrix/client/v3/user").respond(200, (path, content) => {
                 expect(path).toEqual(`${hsUrl}/_matrix/client/v3/user/${encodeURIComponent(userId)}/filter`);
                 expect(content).toMatchObject(filter);
-                client.stop(); // avoid a sync early
                 return { filter_id: filterId };
             });
 
+            // noinspection TypeScriptValidateJSTypes
+            http.when("GET", "/_matrix/client/v3/sync").respond(200, (path, content, req) => {
+                expect(req.queryParams.filter).toBe(filterId);
+                client.stop();
+                return { next_batch: "123" };
+            });
+
             await Promise.all([client.start(filter), http.flushAllExpected()]);
             expect(getFilterFn.callCount).toBe(1);
             expect(setFilterFn.callCount).toBe(1);
@@ -3340,6 +3352,21 @@ describe('MatrixClient', () => {
 
             await Promise.all([client.leaveRoom(roomId), http.flushAllExpected()]);
         });
+        it('should include a reason if provided', async () => {
+            const { client, http, hsUrl } = createTestClient();
+
+            const roomId = "!testing:example.org";
+            const reason = "I am done testing here";
+
+            // noinspection TypeScriptValidateJSTypes
+            http.when("POST", "/_matrix/client/v3/rooms").respond(200, (path, content) => {
+                expect(content).toEqual({ reason });
+                expect(path).toEqual(`${hsUrl}/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/leave`);
+                return {};
+            });
+
+            await Promise.all([client.leaveRoom(roomId, reason), http.flushAllExpected()]);
+        });
     });
 
     describe('forgetRoom', () => {