WIP: postgres: create databases for all services

If a service is enabled, a database for it is created in postgres with a uniqque password. The service can then use this database for data storage instead of relying on sqlite.

group_vars/matrix_servers

@@ -889,6 +889,70 @@ matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
+matrix_postgres_additional_databases: |
+  {{
+    ([{
+      name: 'matrix_appservice_discord',
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+    }] if matrix_appservice_discord_enabled else [])
+    + ([{
+      name: 'matrix_appservice_slack'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+    }] if matrix_appservice_slack_enabled else [])
+    + ([{
+      name: 'matrix_appservice_irc'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+    }] if matrix_appservice_irc_enabled else [])
+    + ([{
+      name: 'mautrix-bridge-facebook'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+    }] if matrix_mautrix_facebook_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_hangouts'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+    }] if matrix_mautrix_hangouts_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_telegram'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+    }] if matrix_mautrix_telegram_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_whatsapp'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+    }] if matrix_mautrix_whatsapp_enabled else [])
+    + ([{
+      name: 'matrix_bridge_sms'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+    }] if matrix_sms_bridge_enabled else [])
+    + ([{
+      name: 'matrix_puppet_skype'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+    }] if matrix_mx_puppet_skype_enabled else [])
+    + ([{
+      name: 'matrix_puppet_slack'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+    }] if matrix_mx_puppet_slack_enabled else [])
+    + ([{
+      name: 'matrix_puppet_twitter'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+    }] if matrix_mx_puppet_twitter_enabled else [])
+    + ([{
+      name: 'matrix_puppet_instagram'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+    ] if matrix_mx_puppet_instagram_enabled else [])
+    + ([{
+      name: 'matrix_puppet_discord'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+    }] if matrix_mx_puppet_discord_enabled else [])
+    + ([{
+      name: 'matrix_puppet_steam'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+    }] if matrix_mx_puppet_steam_enabled else [])
+    + ([{
+      name: 'matrix_dimension'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+    }] if matrix_dimension_enabled else [])
+  }}
+
 ######################################################################
 #
 # /matrix-postgres

roles/matrix-bridge-appservice-discord/templates/config.yaml.j2

@@ -58,8 +58,8 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  filename: "/data/discord.db"
-  # connString: "postgresql://user:password@localhost/database_name"
+  #filename: "/data/discord.db"
+  connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord"
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"

roles/matrix-bridge-appservice-irc/templates/config.yaml.j2

@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: "nedb"
+  engine: "postgres"
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: "nedb:///data"
+  connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"

roles/matrix-bridge-appservice-slack/templates/config.yaml.j2

@@ -10,5 +10,8 @@ homeserver:
     server_name: "{{ matrix_domain }}"
 
 dbdir: "/data"
+db:
+    engine: "postgres"
+    connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"

roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2

@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-facebook.db
+    database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     public:

roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2

@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-hangouts.db
+    database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts
 
     # The unique ID of this appservice.
     id: hangouts

roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2

@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-telegram.db
+    database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in

roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2

@@ -23,7 +23,7 @@ appservice:
         # The database URI.
         #   SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
         #   Postgres: Connection string. For example, postgres://user:password@host/database
-        uri: mautrix-whatsapp.db
+        uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp
         # Maximum number of connections. Mostly relevant for Postgres.
         max_open_conns: 20
         max_idle_conns: 2

roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2

@@ -105,10 +105,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output

roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2

@@ -49,10 +49,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output

roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2

@@ -73,10 +73,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 provisioning:
   # Regex of Matrix IDs allowed to use the puppet bridge

roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2

@@ -63,10 +63,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output

roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2

@@ -66,10 +66,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output

roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2

@@ -59,10 +59,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output

roles/matrix-dimension/templates/config.yaml.j2

@@ -44,7 +44,7 @@ widgetBlacklist:
 
 # Where the database for Dimension is
 database:
-  file: "dimension.db"
+  uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension"
 
 # Display settings that apply to self-hosted go-neb instances
 goneb:

roles/matrix-postgres/tasks/setup_postgres.yml

@@ -162,3 +162,33 @@
     - matrix-change-user-admin-status
     - matrix-postgres-update-user-password-hash
   when: "not matrix_postgres_enabled|bool"
+
+# Create additional databases
+- name: Retrieve IP of postgres container
+  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  register: matirx_postgres_container_ip
+
+- name: Create additional users in postgres
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.pass }}"
+    login_host: "{{ matrx_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+    login_db: "{{ matrix_postgres_db_name }}"
+  loop: matrix_postgres_additional_databases
+  when: matrix_postgres_enabed|bool
+
+- name: Create additional users in postgres
+  postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.name }}"
+    lc_ctype: 'C'
+    lc_collate: 'C'
+    login_host: "{{ matrx_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+  loop: matrix_postgres_additional_databases
+  when: matrix_postgres_enabled|bool