Merge pull request #740 from jdreichmann/postgres-per-default

postgres: create databases for all services

roles/matrix-bot-matrix-reminder-bot/defaults/main.yml

@@ -21,6 +21,34 @@ matrix_bot_matrix_reminder_bot_systemd_required_services_list: ['docker.service'
 matrix_bot_matrix_reminder_bot_systemd_wanted_services_list: []
 
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_bot_matrix_reminder_bot_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_bot_matrix_reminder_bot_database_*` variables
+matrix_bot_matrix_reminder_bot_database_engine: 'sqlite'
+
+matrix_bot_matrix_reminder_bot_sqlite_database_path_local: "{{ matrix_bot_matrix_reminder_bot_data_path }}/bot.db"
+matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container: "/data/bot.db"
+
+matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot'
+matrix_bot_matrix_reminder_bot_database_password: 'some-password'
+matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres'
+matrix_bot_matrix_reminder_bot_database_port: 5432
+matrix_bot_matrix_reminder_bot_database_name: 'matrix_reminder_bot'
+
+matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_name }}'
+
+matrix_bot_matrix_reminder_bot_storage_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container),
+		'postgres': matrix_bot_matrix_reminder_bot_database_connection_string,
+	}[matrix_bot_matrix_reminder_bot_database_engine]
+}}"
+
+
 # The bot's username. This user needs to be created manually beforehand.
 # Also see `matrix_bot_matrix_reminder_bot_user_password`.
 matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: "bot.matrix-reminder-bot"

roles/matrix-bot-matrix-reminder-bot/tasks/main.yml

@@ -8,7 +8,14 @@
     - setup-all
     - setup-bot-matrix-reminder-bot
 
-- import_tasks: "{{ role_path }}/tasks/setup.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_bot_matrix_reminder_bot_enabled|bool"
+  tags:
+    - setup-all
+    - setup-bot-matrix-reminder-bot
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_bot_matrix_reminder_bot_enabled|bool"
   tags:
     - setup-all
     - setup-bot-matrix-reminder-bot

roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml

@@ -0,0 +1,73 @@
+---
+
+- set_fact:
+    matrix_bot_matrix_reminder_bot_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+      register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+              dst: "{{ matrix_bot_matrix_reminder_bot_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_bot_matrix_reminder_bot_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_bot_matrix_reminder_bot_requires_restart: true
+      when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'"
+
+- name: Ensure matrix-reminder-bot paths exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true }
+    - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true }
+    - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true }
+  when: "item.when|bool"
+
+- name: Ensure matrix-reminder-bot image is pulled
+  docker_image:
+    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}"
+
+- name: Ensure matrix-reminder-bot config installed
+  copy:
+    content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+    mode: 0644
+  register: matrix_bot_matrix_reminder_bot_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service restarted, if necessary
+  service:
+    name: "matrix-bot-matrix-reminder-bot.service"
+    state: restarted
+  when: "matrix_bot_matrix_reminder_bot_requires_restart|bool"

roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml

@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-matrix-reminder-bot service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+  register: matrix_bot_matrix_reminder_bot_service_stat
+
+- name: Ensure matrix-matrix-reminder-bot is stopped
+  service:
+    name: matrix-matrix-reminder-bot
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+    state: absent
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure Matrix matrix-reminder-bot paths don't exist
+  file:
+    path: "{{ matrix_bot_matrix_reminder_bot_base_path }}"
+    state: absent
+
+- name: Ensure matrix-reminder-bot Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
+    state: absent

roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2

@@ -23,7 +23,7 @@ storage:
   # For Postgres, this would look like:
   #     database: "postgres://username:password@localhost/dbname?sslmode=disable"
   #database: "postgres://matrix-reminder-bot:remindme@localhost/matrix-reminder-bot?sslmode=disable"
-  database: "sqlite:///data/bot.db"
+  database: {{ matrix_bot_matrix_reminder_bot_storage_database|to_json }}
   # The path to a directory for internal bot storage
   # containing encryption keys, sync tokens, etc.
   store_path: "/data/store"

roles/matrix-bridge-appservice-discord/defaults/main.yml

@@ -41,6 +41,30 @@ matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_appservice_discord_postgres_*` variables
+matrix_appservice_discord_database_engine: 'sqlite'
+
+matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db"
+matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
+
+matrix_appservice_discord_database_username: 'matrix_appservice_discord'
+matrix_appservice_discord_database_password: 'some-password'
+matrix_appservice_discord_database_hostname: 'matrix-postgres'
+matrix_appservice_discord_database_port: 5432
+matrix_appservice_discord_database_name: 'matrix_appservice_discord'
+
+# These 2 variables are what actually ends up in the bridge configuration.
+# It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
+matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_name }}'
+
+
 # Tells whether the bot should make use of "Privileged Gateway Intents".
 #
 # Enabling this means that you need to enable it for the bot (Discord application) as well,

roles/matrix-bridge-appservice-discord/tasks/setup_install.yml

@@ -1,5 +1,31 @@
 ---
 
+- set_fact:
+    matrix_appservice_discord_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+      register: matrix_appservice_discord_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+              dst: "{{ matrix_appservice_discord_database_connString }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_appservice_discord_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-appservice-discord.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_discord_requires_restart: true
+      when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_appservice_discord_database_engine == 'postgres'"
+
 - name: Ensure Appservice Discord image is pulled
   docker_image:
     name: "{{ matrix_appservice_discord_docker_image }}"
@@ -80,3 +106,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_discord_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-discord.service restarted, if necessary
+  service:
+    name: "matrix-appservice-discord.service"
+    state: restarted
+  when: "matrix_appservice_discord_requires_restart|bool"

roles/matrix-bridge-appservice-discord/tasks/validate_config.yml

@@ -20,3 +20,7 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_discord_container_http_host_bind_port>'}
+
+- name: Require a valid database engine
+  fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'"
+  when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']"

roles/matrix-bridge-appservice-discord/templates/config.yaml.j2

@@ -58,8 +58,11 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  filename: "/data/discord.db"
-  # connString: "postgresql://user:password@localhost/database_name"
+  {% if matrix_appservice_discord_database_engine == 'sqlite' %}
+  filename: {{ matrix_appservice_discord_database_filename|to_json }}
+  {% else %}
+  connString: {{ matrix_appservice_discord_database_connString|to_json }}
+  {% endif %}
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"

roles/matrix-bridge-appservice-irc/defaults/main.yml

@@ -16,6 +16,25 @@ matrix_appservice_irc_homeserver_domain: '{{ matrix_domain }}'
 matrix_appservice_irc_homeserver_enablePresence: true
 matrix_appservice_irc_appservice_address: 'http://matrix-appservice-irc:9999'
 
+matrix_appservice_irc_database_engine: nedb
+matrix_appservice_irc_database_username: matrix_appservice_irc
+matrix_appservice_irc_database_password: ~
+matrix_appservice_irc_database_hostname: 'matrix-postgres'
+matrix_appservice_irc_database_port: 5432
+matrix_appservice_irc_database_name: matrix_appservice_irc
+
+# This is just the Postgres connection string, if Postgres is used.
+# Naming clashes with `matrix_appservice_irc_database_connectionString` somewhat.
+matrix_appservice_irc_database_connection_string: 'postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable'
+
+# This is what actually goes into `database.connectionString` for the bridge.
+matrix_appservice_irc_database_connectionString: "{{
+	{
+		'nedb': 'nedb:///data',
+		'postgres': matrix_appservice_irc_database_connection_string,
+	}[matrix_appservice_irc_database_engine]
+}}"
+
 matrix_appservice_irc_ircService_servers: []
 
 # Example of `matrix_appservice_irc_ircService_servers` with one server (and all its options):