postgres: create databases for all services

roles/matrix-bridge-appservice-discord/defaults/main.yml

@@ -41,6 +41,30 @@ matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_appservice_discord_postgres_*` variables
+matrix_appservice_discord_database_engine: 'sqlite'
+
+matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db"
+matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
+
+matrix_appservice_discord_database_username: 'matrix_appservice_discord'
+matrix_appservice_discord_database_password: 'some-password'
+matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
+matrix_appservice_discord_database_port: 5432
+matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
+
+# These 2 variables are what actually ends up in the bridge configuration.
+# It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
+matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
+
+
 # Tells whether the bot should make use of "Privileged Gateway Intents".
 #
 # Enabling this means that you need to enable it for the bot (Discord application) as well,

roles/matrix-bridge-appservice-discord/tasks/setup_install.yml

@@ -1,5 +1,25 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+    register: matrix_appservice_discord_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_appservice_discord_database_engine == 'postgres'"
+
 - name: Ensure Appservice Discord image is pulled
   docker_image:
     name: "{{ matrix_appservice_discord_docker_image }}"

roles/matrix-bridge-appservice-discord/tasks/validate_config.yml

@@ -20,3 +20,7 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_discord_container_http_host_bind_port>'}
+
+- name: Require a valid database engine
+  fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'"
+  when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']"

roles/matrix-bridge-appservice-discord/templates/config.yaml.j2

@@ -58,8 +58,11 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  filename: "/data/discord.db"
-  # connString: "postgresql://user:password@localhost/database_name"
+  {% if matrix_appservice_discord_database_engine == 'sqlite' %}
+  filename: {{ matrix_appservice_discord_database_filename|to_json }}
+  {% else %}
+  connString: {{ matrix_appservice_discord_database_connString|to_json }}
+  {% endif %}
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"

roles/matrix-bridge-appservice-irc/templates/config.yaml.j2

@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: "nedb"
+  engine: "postgres"
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: "nedb:///data"
+  connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"

roles/matrix-bridge-appservice-slack/templates/config.yaml.j2

@@ -10,5 +10,8 @@ homeserver:
     server_name: "{{ matrix_domain }}"
 
 dbdir: "/data"
+db:
+    engine: "postgres"
+    connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"

roles/matrix-bridge-mautrix-facebook/defaults/main.yml

@@ -32,6 +32,35 @@ matrix_mautrix_facebook_systemd_wanted_services_list: []
 matrix_mautrix_facebook_appservice_token: ''
 matrix_mautrix_facebook_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_facebook_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_facebook_postgres_*` variables
+matrix_mautrix_facebook_database_engine: 'sqlite'
+
+matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db"
+matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db"
+
+matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_password: 'some-password'
+matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_facebook_database_port: 5432
+matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
+
+matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
+
+matrix_mautrix_facebook_appservice_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container),
+		'postgres': matrix_mautrix_facebook_postgres_connection_string,
+	}[matrix_mautrix_facebook_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_facebook_login_shared_secret: ''
 

roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml

@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+    register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_facebook_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Facebook image is pulled
   docker_image:
     name: "{{ matrix_mautrix_facebook_docker_image }}"

roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2

@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-facebook.db
+    database: {{ matrix_mautrix_facebook_appservice_database|to_json }}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     public:

roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2

@@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-fac
 			--log-driver=none \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
 			-v {{ matrix_mautrix_facebook_config_path }}:/config:z \
 			-v {{ matrix_mautrix_facebook_data_path }}:/data:z \
 			{{ matrix_mautrix_facebook_docker_image }} \

roles/matrix-bridge-mautrix-hangouts/defaults/main.yml

@@ -39,6 +39,35 @@ matrix_mautrix_hangouts_systemd_wanted_services_list: []
 matrix_mautrix_hangouts_appservice_token: ''
 matrix_mautrix_hangouts_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_hangouts_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_hangouts_postgres_*` variables
+matrix_mautrix_hangouts_database_engine: 'sqlite'
+
+matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db"
+matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db"
+
+matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_password: 'some-password'
+matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_hangouts_database_port: 5432
+matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
+
+matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
+
+matrix_mautrix_hangouts_appservice_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
+		'postgres': matrix_mautrix_hangouts_postgres_connection_string,
+	}[matrix_mautrix_hangouts_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_hangouts_login_shared_secret: ''
 

roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml

@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+    register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Hangouts image is pulled
   docker_image:
     name: "{{ matrix_mautrix_hangouts_docker_image }}"

roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2

@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-hangouts.db
+    database: {{ matrix_mautrix_hangouts_appservice_database|to_json }}
 
     # The unique ID of this appservice.
     id: hangouts

roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2

@@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-han
 			--log-driver=none \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
 			-v {{ matrix_mautrix_hangouts_config_path }}:/config:z \
 			-v {{ matrix_mautrix_hangouts_data_path }}:/data:z \
 			{{ matrix_mautrix_hangouts_docker_image }} \

roles/matrix-bridge-mautrix-telegram/defaults/main.yml

@@ -43,6 +43,35 @@ matrix_mautrix_telegram_systemd_wanted_services_list: []
 matrix_mautrix_telegram_appservice_token: ''
 matrix_mautrix_telegram_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_telegram_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_telegram_postgres_*` variables
+matrix_mautrix_telegram_database_engine: 'sqlite'
+
+matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
+matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db"
+
+matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_password: 'some-password'
+matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_telegram_database_port: 5432
+matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
+
+matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
+
+matrix_mautrix_telegram_appservice_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container),
+		'postgres': matrix_mautrix_telegram_postgres_connection_string,
+	}[matrix_mautrix_telegram_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_telegram_login_shared_secret: ''
 

roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml

@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+    register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_telegram_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Telegram image is pulled
   docker_image:
     name: "{{ matrix_mautrix_telegram_docker_image }}"

roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2

@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-telegram.db
+    database: {{ matrix_mautrix_telegram_appservice_database|to_json }}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in