Appservice.ts

import * as express from "express";
import { EventEmitter } from "events";
import * as morgan from "morgan";
import * as LRU from "lru-cache";
import { stringify } from "querystring";

import { Intent } from "./Intent";
import {
    AppserviceJoinRoomStrategy,
    EncryptedRoomEvent,
    EventKind,
    IAppserviceCryptoStorageProvider,
    IAppserviceStorageProvider,
    IJoinRoomStrategy,
    IPreprocessor,
    LogService,
    MatrixClient,
    MemoryStorageProvider,
    Metrics,
    OTKAlgorithm,
    redactObjectForLogging,
    UserID,
} from "..";
import { MatrixBridge } from "./MatrixBridge";
import { IApplicationServiceProtocol } from "./http_responses";

const EDU_ANNOTATION_KEY = "io.t2bot.sdk.bot.type";

enum EduAnnotation {
    ToDevice = "to_device",
    Ephemeral = "ephemeral",
}

/**
 * Represents an application service's registration file. This is expected to be
 * loaded from another source, such as a YAML file.
 * @category Application services
 */
export interface IAppserviceRegistration {
    /**
     * Optional ID for the appplication service. Used by homeservers to track which application
     * service registers what.
     */
    id?: string;

    /**
     * Optional URL at which the application service can be contacted.
     */
    url?: string;

    /**
     * The token the application service uses to communicate with the homeserver.
     */
    as_token: string;

    /**
     * The token the homeserver uses to communicate with the application service.
     */
    hs_token: string;

    /**
     * The application service's own localpart (eg: "_irc_bot" in the user ID "@_irc_bot:domain.com")
     */
    sender_localpart: string;

    /**
     * The various namespaces the application service can support.
     */
    namespaces: {
        /**
         * The user namespaces the application service is requesting.
         */
        users: {
            /**
             * Whether or not the application service holds an exclusive lock on the namespace. This
             * means that no other user on the homeserver may register users that match this namespace.
             */
            exclusive: boolean;

            /**
             * The regular expression that the homeserver uses to determine if a user is in this namespace.
             */
            regex: string;
        }[];

        /**
         * The room namespaces the application service is requesting. This is not for alises.
         */
        rooms: {
            /**
             * Whether or not the application service holds an exclusive lock on the namespace.
             */
            exclusive: boolean;

            /**
             * The regular expression that the homeserver uses to determine if a user is in this namespace.
             */
            regex: string;
        }[];

        /**
         * The room alias namespaces the application service is requesting.
         */
        aliases: {
            /**
             * Whether or not the application service holds an exclusive lock on the namespace. This means
             * that no other user on the homeserver may register aliases that match this namespace.
             */
            exclusive: boolean;

            /**
             * The regular expression that the homeserver uses to determine if an alias is in this namespace.
             */
            regex: string;
        }[];
    };

    /**
     * The protocols the application service supports. Optional.
     */
    protocols?: string[];

    /**
     * If the application service is rate limited by the homeserver. Optional.
     */
    rate_limited?: boolean;

    /**
     * **Experimental**
     *
     * Should the application service receive ephemeral events from the homeserver. Optional.
     * @see https://github.com/matrix-org/matrix-doc/pull/2409
     */
    "de.sorunome.msc2409.push_ephemeral"?: boolean;

    // not interested in other options
}

/**
 * General options for the application service
 * @category Application services
 */
export interface IAppserviceOptions {
    /**
     * The port to listen for requests from the homeserver on.
     */
    port: number;

    /**
     * The bind address to listen for requests on.
     */
    bindAddress: string;

    /**
     * The name of the homeserver, as presented over federation (eg: "matrix.org")
     */
    homeserverName: string;

    /**
     * The URL to the homeserver's client server API (eg: "https://matrix.org")
     */
    homeserverUrl: string;

    /**
     * The storage provider to use for this application service.
     */
    storage?: IAppserviceStorageProvider;

    /**
     * The storage provider to use for setting up encryption. Encryption will be
     * disabled for all intents and the appservice if not configured.
     */
    cryptoStorage?: IAppserviceCryptoStorageProvider;

    /**
     * The registration for this application service.
     */
    registration: IAppserviceRegistration;

    /**
     * The join strategy to use for all intents, if any.
     */
    joinStrategy?: IJoinRoomStrategy;

    /**
     * Options for how Intents are handled.
     */
    intentOptions?: {
        /**
         * The maximum number of intents to keep cached. Defaults to 10 thousand.
         */
        maxCached?: number;

        /**
         * The maximum age in milliseconds to keep an Intent around for, provided
         * the maximum number of intents has been reached. Defaults to 60 minutes.
         */
        maxAgeMs?: number;

        /**
         * If false (default), crypto will not be automatically set up for all intent
         * instances - it will need to be manually enabled with
         * `await intent.enableEncryption()`.
         *
         * If true, crypto will be automatically set up.
         *
         * Note that the appservice bot account is considered an intent.
         */
        encryption?: boolean;
    };
}

/**
 * Represents an application service. This provides helper utilities such as tracking
 * of user intents (clients that are aware of their membership in rooms).
 * @category Application services
 */
export class Appservice extends EventEmitter {
    /**
     * The metrics instance for this appservice. This will raise all metrics
     * from this appservice instance as well as any intents/MatrixClients created
     * by the appservice.
     */
    public readonly metrics: Metrics = new Metrics();

    private readonly userPrefix: string | null;
    private readonly aliasPrefix: string | null;
    private readonly registration: IAppserviceRegistration;
    private readonly storage: IAppserviceStorageProvider;
    private readonly cryptoStorage: IAppserviceCryptoStorageProvider;
    private readonly bridgeInstance = new MatrixBridge(this);

    private app = express();
    private appServer: any;
    private intentsCache: LRU<string, Intent>;
    private eventProcessors: { [eventType: string]: IPreprocessor[] } = {};
    private pendingTransactions: { [txnId: string]: Promise<any> } = {};

    /**
     * Creates a new application service.
     * @param {IAppserviceOptions} options The options for the application service.
     */
    constructor(private options: IAppserviceOptions) {
        super();

        options.joinStrategy = new AppserviceJoinRoomStrategy(options.joinStrategy, this);

        if (!options.intentOptions) options.intentOptions = {};
        if (!options.intentOptions.maxAgeMs) options.intentOptions.maxAgeMs = 60 * 60 * 1000;
        if (!options.intentOptions.maxCached) options.intentOptions.maxCached = 10000;

        this.intentsCache = new LRU({
            max: options.intentOptions.maxCached,
            ttl: options.intentOptions.maxAgeMs,
        });

        this.registration = options.registration;

        // If protocol is not defined, define an empty array.
        if (!this.registration.protocols) {
            this.registration.protocols = [];
        }

        this.storage = options.storage || new MemoryStorageProvider();
        options.storage = this.storage;
        this.cryptoStorage = options.cryptoStorage;

        this.app.use(express.json({ limit: Number.MAX_SAFE_INTEGER })); // disable limits, use a reverse proxy
        morgan.token('url-safe', (req: express.Request) =>
            `${req.path}?${stringify(redactObjectForLogging(req.query ?? {}))}`,
        );

        this.app.use(morgan(
            // Same as "combined", but with sensitive values removed from requests.
            ':remote-addr - :remote-user [:date[clf]] ":method :url-safe HTTP/:http-version" :status :res[content-length] ":referrer" ":user-agent"',
            { stream: { write: LogService.info.bind(LogService, 'Appservice') } },
        ));

        // ETag headers break the tests sometimes, and we don't actually need them anyways for
        // appservices - none of this should be cached.
        this.app.set('etag', false);

        this.app.get("/users/:userId", this.onUser.bind(this));
        this.app.get("/rooms/:roomAlias", this.onRoomAlias.bind(this));
        this.app.put("/transactions/:txnId", this.onTransaction.bind(this));
        this.app.get("/_matrix/app/v1/users/:userId", this.onUser.bind(this));
        this.app.get("/_matrix/app/v1/rooms/:roomAlias", this.onRoomAlias.bind(this));
        this.app.put("/_matrix/app/v1/transactions/:txnId", this.onTransaction.bind(this));
        this.app.get("/_matrix/app/v1/thirdparty/protocol/:protocol", this.onThirdpartyProtocol.bind(this));
        this.app.get("/_matrix/app/v1/thirdparty/user/:protocol", this.onThirdpartyUser.bind(this));
        this.app.get("/_matrix/app/v1/thirdparty/user", this.onThirdpartyUser.bind(this));
        this.app.get("/_matrix/app/v1/thirdparty/location/:protocol", this.onThirdpartyLocation.bind(this));
        this.app.get("/_matrix/app/v1/thirdparty/location", this.onThirdpartyLocation.bind(this));

        // Everything else can 404

        if (!this.registration.namespaces || !this.registration.namespaces.users || this.registration.namespaces.users.length === 0) {
            throw new Error("No user namespaces in registration");
        }
        if (this.registration.namespaces.users.length !== 1) {
            throw new Error("Too many user namespaces registered: expecting exactly one");
        }

        const userPrefix = (this.registration.namespaces.users[0].regex || "").split(":")[0];
        if (!userPrefix.endsWith(".*") && !userPrefix.endsWith(".+")) {
            this.userPrefix = null;
        } else {
            this.userPrefix = userPrefix.substring(0, userPrefix.length - 2); // trim off the .* part
        }

        if (!this.registration.namespaces?.aliases || this.registration.namespaces.aliases.length !== 1) {
            this.aliasPrefix = null;
        } else {
            this.aliasPrefix = (this.registration.namespaces.aliases[0].regex || "").split(":")[0];
            if (!this.aliasPrefix.endsWith(".*") && !this.aliasPrefix.endsWith(".+")) {
                this.aliasPrefix = null;
            } else {
                this.aliasPrefix = this.aliasPrefix.substring(0, this.aliasPrefix.length - 2); // trim off the .* part
            }
        }
    }

    /**
     * Gets the express app instance which is serving requests. Not recommended for
     * general usage, but may be used to append routes to the web server.
     */
    public get expressAppInstance() {
        return this.app;
    }

    /**
     * Gets the bridge-specific APIs for this application service.
     */
    public get bridge(): MatrixBridge {
        return this.bridgeInstance;
    }

    /**
     * Get the application service's "bot" user ID (the sender_localpart).
     */
    public get botUserId(): string {
        return this.getUserId(this.registration.sender_localpart);
    }

    /**
     * Get the application service's "bot" Intent (the sender_localpart).
     * @returns {Intent} The intent for the application service itself.
     */
    public get botIntent(): Intent {
        return this.getIntentForUserId(this.botUserId);
    }

    /**
     * Get the application service's "bot" MatrixClient (the sender_localpart).
     * Normally the botIntent should be used to ensure that the bot user is safely
     * handled.
     * @returns {MatrixClient} The client for the application service itself.
     */
    public get botClient(): MatrixClient {
        return this.botIntent.underlyingClient;
    }

    /**
     * Starts the application service, opening the bind address to begin processing requests.
     * @returns {Promise<void>} resolves when started
     */
    public begin(): Promise<void> {
        return new Promise<void>((resolve, reject) => {
            this.appServer = this.app.listen(this.options.port, this.options.bindAddress, () => resolve());
        }).then(async () => {
            if (this.options.intentOptions?.encryption) {
                await this.botIntent.enableEncryption();
            } else {
                await this.botIntent.ensureRegistered();
            }
        });
    }

    /**
     * Stops the application service, freeing the web server.
     */
    public stop(): void {
        if (!this.appServer) return;
        this.appServer.close();
    }

    /**
     * Gets an intent for a given localpart. The user ID will be formed with the domain name given
     * in the constructor.
     * @param localpart The localpart to get an Intent for.
     * @returns {Intent} An Intent for the user.
     */
    public getIntent(localpart: string): Intent {
        return this.getIntentForUserId(this.getUserId(localpart));
    }

    /**
     * Gets a full user ID for a given localpart. The user ID will be formed with the domain name given
     * in the constructor.
     * @param localpart The localpart to get a user ID for.
     * @returns {string} The user's ID.
     */
    public getUserId(localpart: string): string {
        return `@${localpart}:${this.options.homeserverName}`;
    }

    /**
     * Gets an Intent for a given user suffix. The prefix is automatically detected from the registration
     * options.
     * @param suffix The user's suffix
     * @returns {Intent} An Intent for the user.
     */
    public getIntentForSuffix(suffix: string): Intent {
        return this.getIntentForUserId(this.getUserIdForSuffix(suffix));
    }

    /**
     * Gets a full user ID for a given suffix. The prefix is automatically detected from the registration
     * options.
     * @param suffix The user's suffix
     * @returns {string} The user's ID.
     */
    public getUserIdForSuffix(suffix: string): string {
        if (!this.userPrefix) {
            throw new Error(`Cannot use getUserIdForSuffix, provided namespace did not include a valid suffix`);
        }
        return `${this.userPrefix}${suffix}:${this.options.homeserverName}`;
    }

    /**
     * Gets an Intent for a given user ID.
     * @param {string} userId The user ID to get an Intent for.
     * @returns {Intent} An Intent for the user.
     */
    public getIntentForUserId(userId: string): Intent {
        let intent: Intent = this.intentsCache.get(userId);
        if (!intent) {
            intent = new Intent(this.options, userId, this);
            this.intentsCache.set(userId, intent);
            if (this.options.intentOptions.encryption) {
                intent.enableEncryption().catch(e => {
                    LogService.error("Appservice", `Failed to set up crypto on intent ${userId}`, e);
                    throw e; // re-throw to cause unhandled exception
                });
            }
        }
        return intent;
    }

    /**
     * Gets the suffix for the provided user ID. If the user ID is not a namespaced
     * user, this will return a falsey value.
     * @param {string} userId The user ID to parse
     * @returns {string} The suffix from the user ID.
     */
    public getSuffixForUserId(userId: string): string {
        if (!this.userPrefix) {
            throw new Error(`Cannot use getUserIdForSuffix, provided namespace did not include a valid suffix`);
        }
        if (!userId || !userId.startsWith(this.userPrefix) || !userId.endsWith(`:${this.options.homeserverName}`)) {
            // Invalid ID
            return null;
        }

        return userId
            .split('')
            .slice(this.userPrefix.length)
            .reverse()
            .slice(this.options.homeserverName.length + 1)
            .reverse()
            .join('');
    }

    /**
     * Determines if a given user ID is namespaced by this application service.
     * @param {string} userId The user ID to check
     * @returns {boolean} true if the user is namespaced, false otherwise
     */
    public isNamespacedUser(userId: string): boolean {
        return userId === this.botUserId ||
            !!this.registration.namespaces?.users.find(({ regex }) =>
                new RegExp(regex).test(userId),
            );
    }

    /**
     * Gets a full alias for a given localpart. The alias will be formed with the domain name given
     * in the constructor.
     * @param localpart The localpart to get an alias for.
     * @returns {string} The alias.
     */
    public getAlias(localpart: string): string {
        return `#${localpart}:${this.options.homeserverName}`;
    }

    /**
     * Gets a full alias for a given suffix. The prefix is automatically detected from the registration
     * options.
     * @param suffix The alias's suffix
     * @returns {string} The alias.
     */
    public getAliasForSuffix(suffix: string): string {
        if (!this.aliasPrefix) {
            throw new Error("Invalid configured alias prefix");
        }
        return `${this.aliasPrefix}${suffix}:${this.options.homeserverName}`;
    }

    /**
     * Gets the localpart of an alias for a given suffix. The prefix is automatically detected from the registration
     * options. Useful for the createRoom endpoint.
     * @param suffix The alias's suffix
     * @returns {string} The alias localpart.
     */
    public getAliasLocalpartForSuffix(suffix: string): string {
        if (!this.aliasPrefix) {
            throw new Error("Invalid configured alias prefix");
        }
        return `${this.aliasPrefix.substr(1)}${suffix}`;
    }

    /**
     * Gets the suffix for the provided alias. If the alias is not a namespaced
     * alias, this will return a falsey value.
     * @param {string} alias The alias to parse
     * @returns {string} The suffix from the alias.
     */
    public getSuffixForAlias(alias: string): string {
        if (!this.aliasPrefix) {
            throw new Error("Invalid configured alias prefix");
        }
        if (!alias || !this.isNamespacedAlias(alias)) {
            // Invalid ID
            return null;
        }

        return alias
            .split('')
            .slice(this.aliasPrefix.length)
            .reverse()
            .slice(this.options.homeserverName.length + 1)
            .reverse()
            .join('');
    }

    /**
     * Determines if a given alias is namespaced by this application service.
     * @param {string} alias The alias to check
     * @returns {boolean} true if the alias is namespaced, false otherwise
     */
    public isNamespacedAlias(alias: string): boolean {
        if (!this.aliasPrefix) {
            throw new Error("Invalid configured alias prefix");
        }
        return alias.startsWith(this.aliasPrefix) && alias.endsWith(":" + this.options.homeserverName);
    }

    /**
     * Adds a preprocessor to the event pipeline. When this appservice encounters an event, it
     * will try to run it through the preprocessors it can in the order they were added.
     * @param {IPreprocessor} preprocessor the preprocessor to add
     */
    public addPreprocessor(preprocessor: IPreprocessor): void {
        if (!preprocessor) throw new Error("Preprocessor cannot be null");

        const eventTypes = preprocessor.getSupportedEventTypes();
        if (!eventTypes) return; // Nothing to do

        for (const eventType of eventTypes) {
            if (!this.eventProcessors[eventType]) this.eventProcessors[eventType] = [];
            this.eventProcessors[eventType].push(preprocessor);
        }
    }

    /**
     * Sets the visibility of a room in the appservice's room directory.
     * @param {string} networkId The network ID to group the room under.
     * @param {string} roomId The room ID to manipulate the visibility of.
     * @param {"public" | "private"} visibility The visibility to set for the room.
     * @return {Promise<any>} resolves when the visibility has been updated.
     */
    public setRoomDirectoryVisibility(networkId: string, roomId: string, visibility: "public" | "private") {
        roomId = encodeURIComponent(roomId);
        networkId = encodeURIComponent(networkId);
        return this.botClient.doRequest("PUT", `/_matrix/client/v3/directory/list/appservice/${networkId}/${roomId}`, null, {
            visibility,
        });
    }

    private async processEphemeralEvent(event: any): Promise<any> {
        if (!event) return event;
        if (!this.eventProcessors[event["type"]]) return event;

        for (const processor of this.eventProcessors[event["type"]]) {
            await processor.processEvent(event, this.botIntent.underlyingClient, EventKind.EphemeralEvent);
        }

        return event;
    }

    private async processEvent(event: any): Promise<any> {
        if (!event) return event;
        if (!this.eventProcessors[event["type"]]) return event;

        for (const processor of this.eventProcessors[event["type"]]) {
            await processor.processEvent(event, this.botIntent.underlyingClient, EventKind.RoomEvent);
        }

        return event;
    }

    private async processMembershipEvent(event: any): Promise<void> {
        if (!event["content"]) return;

        const domain = new UserID(event['state_key']).domain;
        const botDomain = new UserID(this.botUserId).domain;
        if (domain !== botDomain) return; // can't be impersonated, so don't try

        // Update the target intent's joined rooms (fixes transition errors with the cache, like join->kick->join)
        const intent = this.getIntentForUserId(event['state_key']);
        await intent.refreshJoinedRooms();

        const targetMembership = event["content"]["membership"];
        if (targetMembership === "join") {
            this.emit("room.join", event["room_id"], event);
            await intent.underlyingClient.crypto?.onRoomJoin(event["room_id"]);
        } else if (targetMembership === "ban" || targetMembership === "leave") {
            this.emit("room.leave", event["room_id"], event);
        } else if (targetMembership === "invite") {
            this.emit("room.invite", event["room_id"], event);
        }
    }

    private isAuthed(req: any): boolean {
        let providedToken = req.query ? req.query["access_token"] : null;
        if (req.headers && req.headers["Authorization"]) {
            const authHeader = req.headers["Authorization"];
            if (!authHeader.startsWith("Bearer ")) providedToken = null;
            else providedToken = authHeader.substring("Bearer ".length);
        }

        return providedToken === this.registration.hs_token;
    }

    private async onTransaction(req: express.Request, res: express.Response): Promise<any> {
        if (!this.isAuthed(req)) {
            res.status(401).json({ errcode: "AUTH_FAILED", error: "Authentication failed" });
            return;
        }

        if (typeof (req.body) !== "object") {
            res.status(400).json({ errcode: "BAD_REQUEST", error: "Expected JSON" });
            return;
        }

        if (!req.body["events"] || !Array.isArray(req.body["events"])) {
            res.status(400).json({ errcode: "BAD_REQUEST", error: "Invalid JSON: expected events" });
            return;
        }

        const txnId = req.params["txnId"];

        if (await Promise.resolve(this.storage.isTransactionCompleted(txnId))) {
            res.status(200).json({});
            return;
        }

        if (this.pendingTransactions[txnId]) {
            try {
                await this.pendingTransactions[txnId];
                res.status(200).json({});
            } catch (e) {
                LogService.error("Appservice", e);
                res.status(500).json({});
            }
            return;
        }

        LogService.info("Appservice", "Processing transaction " + txnId);
        // eslint-disable-next-line no-async-promise-executor
        this.pendingTransactions[txnId] = new Promise<void>(async (resolve) => {
            // Process all the crypto stuff first to ensure that future transactions (if not this one)
            // will decrypt successfully. We start with EDUs because we need structures to put counts
            // and such into in a later stage, and EDUs are independent of crypto.

            const byUserId: {
                [userId: string]: {
                    counts: Record<string, Number>;
                    toDevice: any[];
                    unusedFallbacks: OTKAlgorithm[];
                };
            } = {};

            const orderedEdus = [];
            if (Array.isArray(req.body["de.sorunome.msc2409.to_device"])) {
                orderedEdus.push(...req.body["de.sorunome.msc2409.to_device"].map(e => ({
                    ...e,
                    unsigned: {
                        ...e['unsigned'],
                        [EDU_ANNOTATION_KEY]: EduAnnotation.ToDevice,
                    },
                })));
            }
            if (Array.isArray(req.body["de.sorunome.msc2409.ephemeral"])) {
                orderedEdus.push(...req.body["de.sorunome.msc2409.ephemeral"].map(e => ({
                    ...e,
                    unsigned: {
                        ...e['unsigned'],
                        [EDU_ANNOTATION_KEY]: EduAnnotation.Ephemeral,
                    },
                })));
            }
            for (let event of orderedEdus) {
                if (event['edu_type']) event['type'] = event['edu_type']; // handle property change during MSC2409's course

                LogService.info("Appservice", `Processing ${event['unsigned'][EDU_ANNOTATION_KEY]} event of type ${event["type"]}`);
                event = await this.processEphemeralEvent(event);

                // These events aren't tied to rooms, so just emit them generically
                this.emit("ephemeral.event", event);

                if (event["type"] === "m.room.encrypted" || event[EDU_ANNOTATION_KEY] === EduAnnotation.ToDevice) {
                    const toUser = event["to_user_id"];
                    const intent = this.getIntentForUserId(toUser);
                    await intent.enableEncryption();

                    if (!byUserId[toUser]) byUserId[toUser] = { counts: null, toDevice: null, unusedFallbacks: null };
                    if (!byUserId[toUser].toDevice) byUserId[toUser].toDevice = [];
                    byUserId[toUser].toDevice.push(event);
                }
            }

            if (this.cryptoStorage) {
                const deviceLists: { changed: string[], removed: string[] } = req.body["org.matrix.msc3202.device_lists"] ?? {
                    changed: [],
                    removed: [],
                };

                if (!deviceLists.changed) deviceLists.changed = [];
                if (!deviceLists.removed) deviceLists.removed = [];

                const otks = req.body["org.matrix.msc3202.device_one_time_key_counts"];
                if (otks) {
                    for (const userId of Object.keys(otks)) {
                        const intent = this.getIntentForUserId(userId);
                        await intent.enableEncryption();
                        const otksForUser = otks[userId][intent.underlyingClient.crypto.clientDeviceId];
                        if (otksForUser) {
                            if (!byUserId[userId]) {
                                byUserId[userId] = {
                                    counts: null,
                                    toDevice: null,
                                    unusedFallbacks: null,
                                };
                            }
                            byUserId[userId].counts = otksForUser;
                        }
                    }
                }

                const fallbacks = req.body["org.matrix.msc3202.device_unused_fallback_key_types"];
                if (fallbacks) {
                    for (const userId of Object.keys(fallbacks)) {
                        const intent = this.getIntentForUserId(userId);
                        await intent.enableEncryption();
                        const fallbacksForUser = fallbacks[userId][intent.underlyingClient.crypto.clientDeviceId];
                        if (Array.isArray(fallbacksForUser) && !fallbacksForUser.includes(OTKAlgorithm.Signed)) {
                            if (!byUserId[userId]) {
                                byUserId[userId] = {
                                    counts: null,
                                    toDevice: null,
                                    unusedFallbacks: null,
                                };
                            }
                            byUserId[userId].unusedFallbacks = fallbacksForUser;
                        }
                    }
                }

                for (const userId of Object.keys(byUserId)) {
                    const intent = this.getIntentForUserId(userId);
                    await intent.enableEncryption();
                    const info = byUserId[userId];
                    const userStorage = this.storage.storageForUser(userId);

                    if (!info.toDevice) info.toDevice = [];
                    if (!info.unusedFallbacks) info.unusedFallbacks = JSON.parse(await userStorage.readValue("last_unused_fallbacks") || "[]");
                    if (!info.counts) info.counts = JSON.parse(await userStorage.readValue("last_counts") || "{}");

                    LogService.info("Appservice", `Updating crypto state for ${userId}`);
                    await intent.underlyingClient.crypto.updateSyncData(info.toDevice, info.counts, info.unusedFallbacks, deviceLists.changed, deviceLists.removed);
                }
            }

            for (let event of req.body["events"]) {
                LogService.info("Appservice", `Processing event of type ${event["type"]}`);
                event = await this.processEvent(event);
                if (event['type'] === 'm.room.encrypted' && this.cryptoStorage) {
                    this.emit("room.encrypted_event", event["room_id"], event);
                    try {
                        const encrypted = new EncryptedRoomEvent(event);
                        const roomId = event['room_id'];
                        try {
                            event = (await this.botClient.crypto.decryptRoomEvent(encrypted, roomId)).raw;
                            event = await this.processEvent(event);
                            this.emit("room.decrypted_event", roomId, event);

                            // For logging purposes: show that the event was decrypted
                            LogService.info("Appservice", `Processing decrypted event of type ${event["type"]}`);
                        } catch (e1) {
                            LogService.warn("Appservice", `Bot client was not able to decrypt ${roomId} ${event['event_id']} - trying other intents`);

                            let tryUserId: string;
                            try {
                                // TODO: This could be more efficient
                                const userIdsInRoom = await this.botClient.getJoinedRoomMembers(roomId);
                                tryUserId = userIdsInRoom.find(u => this.isNamespacedUser(u));
                            } catch (e) {
                                LogService.error("Appservice", "Failed to get members of room - cannot decrypt message");
                            }

                            if (tryUserId) {
                                const intent = this.getIntentForUserId(tryUserId);

                                event = (await intent.underlyingClient.crypto.decryptRoomEvent(encrypted, roomId)).raw;
                                event = await this.processEvent(event);
                                this.emit("room.decrypted_event", roomId, event);

                                // For logging purposes: show that the event was decrypted
                                LogService.info("Appservice", `Processing decrypted event of type ${event["type"]}`);
                            } else {
                                // noinspection ExceptionCaughtLocallyJS
                                throw e1;
                            }
                        }
                    } catch (e) {
                        LogService.error("Appservice", `Decryption error on ${event['room_id']} ${event['event_id']}`, e);
                        this.emit("room.failed_decryption", event['room_id'], event, e);
                    }
                }
                this.emit("room.event", event["room_id"], event);
                if (event['type'] === 'm.room.message') {
                    this.emit("room.message", event["room_id"], event);
                }
                if (event['type'] === 'm.room.member' && this.isNamespacedUser(event['state_key'])) {
                    await this.processMembershipEvent(event);
                }
                if (event['type'] === 'm.room.tombstone' && event['state_key'] === '') {
                    this.emit("room.archived", event['room_id'], event);
                }
                if (event['type'] === 'm.room.create' && event['state_key'] === '' && event['content'] && event['content']['predecessor']) {
                    this.emit("room.upgraded", event['room_id'], event);
                }
            }

            resolve();
        });

        try {
            await this.pendingTransactions[txnId];
            await Promise.resolve(this.storage.setTransactionCompleted(txnId));
            res.status(200).json({});
        } catch (e) {
            LogService.error("Appservice", e);
            res.status(500).json({});
        }
    }

    private async onUser(req: express.Request, res: express.Response): Promise<any> {
        if (!this.isAuthed(req)) {
            res.status(401).json({ errcode: "AUTH_FAILED", error: "Authentication failed" });
            return;
        }

        const userId = req.params["userId"];
        this.emit("query.user", userId, async (result) => {
            if (result.then) result = await result;
            if (result === false) {
                res.status(404).json({ errcode: "USER_DOES_NOT_EXIST", error: "User not created" });
            } else {
                const intent = this.getIntentForUserId(userId);
                await intent.ensureRegistered();
                if (result.display_name) await intent.underlyingClient.setDisplayName(result.display_name);
                if (result.avatar_mxc) await intent.underlyingClient.setAvatarUrl(result.avatar_mxc);
                res.status(200).json(result); // return result for debugging + testing
            }
        });
    }

    private async onRoomAlias(req: express.Request, res: express.Response): Promise<any> {
        if (!this.isAuthed(req)) {
            res.status(401).json({ errcode: "AUTH_FAILED", error: "Authentication failed" });
            return;
        }

        const roomAlias = req.params["roomAlias"];
        this.emit("query.room", roomAlias, async (result) => {
            if (result.then) result = await result;
            if (result === false) {
                res.status(404).json({ errcode: "ROOM_DOES_NOT_EXIST", error: "Room not created" });
            } else {
                const intent = this.botIntent;
                await intent.ensureRegistered();

                result["room_alias_name"] = roomAlias.substring(1).split(':')[0];
                result["__roomId"] = await intent.underlyingClient.createRoom(result);

                res.status(200).json(result); // return result for debugging + testing
            }
        });
    }

    private onThirdpartyProtocol(req: express.Request, res: express.Response) {
        if (!this.isAuthed(req)) {
            res.status(401).json({ errcode: "AUTH_FAILED", error: "Authentication failed" });
            return;
        }

        const protocol = req.params["protocol"];
        if (!this.registration.protocols.includes(protocol)) {
            res.status(404).json({
                errcode: "PROTOCOL_NOT_HANDLED",
                error: "Protocol is not handled by this appservice",
            });
            return;
        }
        this.emit("thirdparty.protocol", protocol, (protocolResponse: IApplicationServiceProtocol) => {
            res.status(200).json(protocolResponse);
        });
    }

    private handleThirdpartyObject(req: express.Request, res: express.Response, objType: string, matrixId?: string) {
        if (!this.isAuthed(req)) {
            res.status(401).json({ errcode: "AUTH_FAILED", error: "Authentication failed" });
            return;
        }

        const protocol = req.params["protocol"];
        const responseFunc = (items: any[]) => {
            if (items && items.length > 0) {
                res.status(200).json(items);
                return;
            }
            res.status(404).json({
                errcode: "NO_MAPPING_FOUND",
                error: "No mappings found",
            });
        };

        // Lookup remote objects(s)
        if (protocol) { // If protocol is given, we are looking up a objects based on fields
            if (!this.registration.protocols.includes(protocol)) {
                res.status(404).json({
                    errcode: "PROTOCOL_NOT_HANDLED",
                    error: "Protocol is not handled by this appservice",
                });
                return;
            }
            // Remove the access_token
            delete req.query.access_token;
            this.emit(`thirdparty.${objType}.remote`, protocol, req.query, responseFunc);
            return;
        } else if (matrixId) { // If a user ID is given, we are looking up a remote objects based on a id
            this.emit(`thirdparty.${objType}.matrix`, matrixId, responseFunc);
            return;
        }

        res.status(400).json({
            errcode: "INVALID_PARAMETERS",
            error: "Invalid parameters given",
        });
    }

    private onThirdpartyUser(req: express.Request, res: express.Response) {
        return this.handleThirdpartyObject(req, res, "user", req.query["userid"] as string);
    }

    private onThirdpartyLocation(req: express.Request, res: express.Response) {
        return this.handleThirdpartyObject(req, res, "location", req.query["alias"] as string);
    }
}