Merge pull request #75 from noelmcloughlin/cluster1

feat(cluster): rabbitmq clustering support
saltstack-formulas · Jun 16, 2021 · 59e830a · 59e830a
2 parents 9517dd3 + 8bc371d
commit 59e830a
Show file tree

Hide file tree

Showing 35 changed files with 380 additions and 228 deletions.
diff --git a/docs/README.rst b/docs/README.rst
@@ -67,7 +67,7 @@ now ``pre-commit`` will run automatically on each ``git commit``. ::
 Special notes
 -------------
 
-None
+The ``rabbitmq`` state does not include ``rabbitmq.config.cluster`` state (initial cluster setup). This formula does not provide any 'forget' state (for inconsistent cluster fixing) during cluster join (could be future feature).
 
 Available states
 ----------------
@@ -99,7 +99,12 @@ This state will install the rabbitmqadmin package only.
 ^^^^^^^^^^^^^^^^^^^
 
 This state will configure the rabbitmq service and has a dependency on ``rabbitmq.install``
-via include list. See ``pillar.example``.
+via include list. It excludes ``rabbitmq.config.cluster`` state
+
+``rabbitmq.config.cluster``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For initial setup this state writes the erlang cookie, joins cluster, and restarts service. The erlang cookie comes from pillar data and must the identical for all cluster members. Join fails if cluster is inconsistent (see rabbitmqctl forget_cluster_node rabbit@somehost).
 
 ``rabbitmq.service``
 ^^^^^^^^^^^^^^^^^^^^

diff --git a/pillar.example b/pillar.example
@@ -2,13 +2,21 @@
 # vim: ft=yaml
 ---
 rabbitmq:
+  cluster:
+    rabbit@locahost:
+      user: rabbit    # 'node' would make more sense here
+      host: localhost  # short hostname of node to join to, not fqdn
+      ram_node: None
+      runas: rabbitmq
+      erlang_cookie:
+        name: /var/lib/rabbitmq/.erlang.cookie
+        value: shared-value-for-all-cluster-members
   pkg:
     # https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.8.14
     use_upstream: repo  # if available (i.e. packagecloud)
   service:
     enabled: true
     running: true
-
   config:
     # see rabbitmq.conf.example from rabbitmq-server github docs
     context: {}
@@ -36,15 +44,16 @@ rabbitmq:
 
   queue:
     my-new-queue:
-      - user: saltstack
-      - passwd: 'password'
-      - durable: true
-      - auto_delete: false
-      - vhost: test_vhost
-      - arguments:
-          - 'x-message-ttl': 8640000
-          - 'x-expires': 8640000
-          - 'x-dead-letter-exchange': 'my-new-exchange'
+      ## note : dict format
+      user: saltstack
+      passwd: password
+      durable: true
+      auto_delete: false
+      vhost: test_vhost
+      arguments:
+        - x-message-ttl: 8640000
+        - x-expires: 8640000
+        - x-dead-letter-exchange: my-new-exchange
 
   exchange:
     my-new-exchange:
@@ -59,13 +68,6 @@ rabbitmq:
           - 'alternate-**exchange': 'amq.fanout'
           - 'test-header': 'testing'
 
-  cluster:
-    rabbit@locahost:
-      - user: rabbit
-      - host: localhost
-      - ram_node: None
-      - runas: root
-
   plugin:
     rabbitmq_management:
       runas: root

diff --git a/rabbitmq/config/clean.sls b/rabbitmq/config/clean.sls
@@ -12,6 +12,7 @@ include:
   - .policy.clean
   - .plugin.clean
   - .upstream.clean
+  - .queue.clean
   - .vhost.clean
   - .user.clean
   - .file.clean

diff --git a/rabbitmq/config/cluster/install.sls b/rabbitmq/config/cluster/install.sls
@@ -10,15 +10,32 @@ include:
   - {{ sls_service_running }}
   - {{ sls_config_user }}
 
-    {% for name, cluster in salt["pillar.get"]("rabbitmq:cluster", {}).items() %}
+    {%- for name, cluster in salt["pillar.get"]("rabbitmq:cluster", {}).items() %}
+        {%- if cluster.host and 'erlang_cookie' in cluster and cluster.erlang_cookie is mapping %}
 
-rabbitmq-config-cluster-join-{{ name }}:
-  rabbitmq_cluster.join:
-    {% for value in cluster %}
-    - {{ value | json }}
-    {% endfor %}
+rabbitmq-config-cluster-{{ name }}-join-{{ cluster.host }}:
+  file.managed:
+    - name: {{ cluster.erlang_cookie.name }}
+    - contents: {{ cluster.erlang_cookie.value }}
+    - mode: 400
+    - user: {{ rabbitmq.config.user }}
+    - group: {{ rabbitmq.config.user }}
+    - makedirs: True
+    - watch_in:
+      - service: rabbitmq-service-running-service-running
+
+            {%- if 'host' in grains and grains.host not in cluster.host %}
+
+  rabbitmq_cluster.joined:
+    - user: {{ cluster.user }}
+    - host: {{ cluster.host }}
+    - ram_node: {{ cluster.ram_node }}
+    - runas: {{ cluster.runas }}
     - require:
-      - service: {{ rabbitmq.service.name }}
+      - file: rabbitmq-config-cluster-{{ name }}-join-{{ cluster.host }}
       - sls: {{ sls_config_user }}
+      - service: rabbitmq-service-running-service-running
 
-    {% endfor %}
+            {%- endif %}
+        {%- endif %}
+    {%- endfor %}
diff --git a/rabbitmq/config/init.sls b/rabbitmq/config/init.sls
@@ -5,7 +5,8 @@ include:
   - .file
   - .user
   - .vhost
+  - .queue
   - .policy
   - .plugin
   - .upstream
-  # .cluster
+  # .cluster should be run once, not always
diff --git a/rabbitmq/config/policy/install.sls b/rabbitmq/config/policy/install.sls
@@ -16,6 +16,6 @@ rabbitmq-config-policy-present-{{ name }}:
     - {{ value | json }}
     {% endfor %}
     - require:
-      - service: {{ rabbitmq.service.name }}
+      - service: rabbitmq-service-running-service-running
 
     {% endfor %}
diff --git a/rabbitmq/config/queue/clean.sls b/rabbitmq/config/queue/clean.sls
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import mapdata as rabbitmq with context %}
+{%- set sls_service_running = tplroot ~ '.service.running' %}
+
+include:
+  - {{ sls_service_running }}
+
+    {% for name, q in salt["pillar.get"]("rabbitmq:queue", {}).items() %}
+
+rabbitmq-config-queue-absent-{{ name }}:
+  cmd.run:
+    - name: /usr/local/sbin/rabbitmqadmin delete queue --vhost={{ q.vhost }} --username={{ q.user }} --password={{ q.passwd }} name={{ name }}  # noqa 204
+    - require:
+      - service: rabbitmq-service-running-service-running
+
+    {% endfor %}
diff --git a/rabbitmq/config/queue/init.sls b/rabbitmq/config/queue/init.sls
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .install
diff --git a/rabbitmq/config/queue/install.sls b/rabbitmq/config/queue/install.sls
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import mapdata as rabbitmq with context %}
+{%- set sls_service_running = tplroot ~ '.service.running' %}
+
+include:
+  - {{ sls_service_running }}
+
+    {% for name, q in salt["pillar.get"]("rabbitmq:queue", {}).items() %}
+
+rabbitmq-config-queue-present-{{ name }}:
+  cmd.run:
+    - name: /usr/local/sbin/rabbitmqadmin declare queue --vhost={{ q.vhost }} --username={{ q.user }} --password={{ q.passwd }} name={{ name }} durable={{ q.durable|to_bool|lower }} auto_delete={{ q.auto_delete|to_bool|lower }}  # noqa 204
+    - require:
+      - service: rabbitmq-service-running-service-running
+
+    {% endfor %}
diff --git a/rabbitmq/config/upstream/install.sls b/rabbitmq/config/upstream/install.sls
@@ -16,6 +16,6 @@ rabbitmq-config-upstream-present-{{ name }}:
     - {{ value | json }}
     {% endfor %}
     - require:
-      - service: {{ rabbitmq.service.name }}
+      - service: rabbitmq-service-running-service-running
 
     {% endfor %}
diff --git a/rabbitmq/config/user/clean.sls b/rabbitmq/config/user/clean.sls
@@ -8,7 +8,7 @@
 include:
   - {{ sls_package_clean }}
 
-    {% for name in rabbitmq.user %}
+    {% for name, user in salt["pillar.get"]("rabbitmq:user", {}).items() %}
 
 rabbitmq-config-user-absent-{{ name }}:
   rabbitmq_user.absent:

diff --git a/rabbitmq/config/user/install.sls b/rabbitmq/config/user/install.sls
@@ -15,9 +15,9 @@ include:
 rabbitmq-config-user-present-{{ name }}:
   rabbitmq_user.present:
     - name: {{ name }}
-    {% for value in user %}
-    - {{ value | yaml }}
-    {% endfor %}
+      {%- for item in user %}
+    - {{ item|json }}
+      {%- endfor %}
     - require:
       - sls: {{ sls_service_running }}
       - sls: {{ sls_config_vhost }}

diff --git a/rabbitmq/config/vhost/install.sls b/rabbitmq/config/vhost/install.sls
@@ -14,6 +14,6 @@ rabbitmq-config-vhost-present-{{ name }}:
   rabbitmq_vhost.present:
     - name: {{ name }}
     - require:
-      - service: {{ rabbitmq.service.name }}
+      - service: rabbitmq-service-running-service-running
 
     {% endfor %}
diff --git a/rabbitmq/package/repo/clean.sls b/rabbitmq/package/repo/clean.sls
@@ -15,7 +15,6 @@ rabbitmq-package-repo-erlang-clean:
   pkgrepo.absent:
     - names:
       - rabbitmq-rabbitmq-erlang
-      - rabbitmq_rabbitmq-erlang
       - rabbitmq-rabbitmq
       - rabbitmq_rabbitmq-server
       - deb https://dl.bintray.com/rabbitmq-erlang/debian {{ salt['grains.get']('oscodename') }} erlang

diff --git a/rabbitmq/parameters/defaults.yaml b/rabbitmq/parameters/defaults.yaml
@@ -4,6 +4,15 @@
 # Set default values.
 ---
 values:
+  cluster:
+    rabbit@locahost:
+      user: rabbit    # 'node' would make more sense here
+      host: localhost  # short hostname of node to join to, not fqdn
+      ram_node: None
+      runas: rabbitmq
+      erlang_cookie:
+        name: /var/lib/rabbitmq/.erlang.cookie
+        value: shared-value-for-all-cluster-members
   pkg:
     name: rabbitmq-server
     # https://github.com/rabbitmq/rabbitmq-server/releases/tag/v3.8.14
@@ -15,16 +24,17 @@ values:
     cleanlist:
       - /usr/local/bin/rabbitmq-env
       - /usr/local/bin/rabbitmq-plugins
+      - /var/run/rabbitmq
   env:
     name: /etc/rabbitmq/rabbitmq-env.conf
     locale_all: null
     context:
       rabbitmq_mnesia_base: /var/lib/rabbitmq
       RABBITMQ_BASE: /var/lib/rabbitmq
   config:
+    context: {}
     name: /etc/rabbitmq/rabbitmq.conf
     # see rabbitmq.conf.example from rabbitmq-server github docs
-    context: {}
   service:
     name: rabbitmq-server
     enabled: true

diff --git a/rabbitmq/service/clean.sls b/rabbitmq/service/clean.sls
@@ -12,5 +12,4 @@ rabbitmq-service-clean-service-dead:
   service.dead:
     - name: {{ rabbitmq.service.name }}
     - enable: False
-    - require_in:
-      - sls: {{ sls_package_clean }}
+    - sig: 'rabbit boot'
diff --git a/rabbitmq/service/running.sls b/rabbitmq/service/running.sls
@@ -20,15 +20,8 @@ rabbitmq-service-running-service-running:
     - enable: True
     - watch:
       - sls: {{ sls_config_file }}
+    - onfail_in:
+      - cmd: rabbitmq-service-running-service-running
   cmd.run:
     - names:
-      - localectl || true
       - journalctl -xe -u {{ rabbitmq.service.name }} || true
-      - systemctl status {{ rabbitmq.service.name }} || true
-      - systemd-analyze blame || true
-      - hostname
-      - hostname -s
-      - hostname -A
-      - ip addr
-    - onfail:
-      - service: rabbitmq-service-running-service-running
diff --git a/test/integration/default/files/_mapdata/amazonlinux-2.yaml b/test/integration/default/files/_mapdata/amazonlinux-2.yaml
@@ -17,10 +17,13 @@ values:
       - x-message-ttl: 8640000
   cluster:
     rabbit@locahost:
-    - user: rabbit
-    - host: localhost
-    - ram_node: None
-    - runas: root
+      erlang_cookie:
+        name: /var/lib/rabbitmq/.erlang.cookie
+        value: shared-value-for-all-cluster-members
+      host: localhost  # short hostname of node to join to, not fqdn
+      ram_node: None
+      runas: rabbitmq
+      user: rabbit    # 'node' would make more sense here
   config:
     context: {}
     name: /etc/rabbitmq.conf
@@ -29,6 +32,7 @@ values:
     cleanlist:
     - /usr/local/bin/rabbitmq-env
     - /usr/local/bin/rabbitmq-plugins
+    - /var/run/rabbitmq
     data: /var/lib/rabbitmq
   env:
     context:
@@ -58,6 +62,7 @@ values:
     - C@rabbitmq
     - Y:G@id
   pkg:
+    deps: []
     name: rabbitmq-server
     use_upstream: repo
   plugin:
@@ -72,15 +77,15 @@ values:
     - definition: '{"ha-mode": "all"}'
   queue:
     my-new-queue:
-    - user: saltstack
-    - passwd: password
-    - durable: true
-    - auto_delete: false
-    - vhost: test_vhost
-    - arguments:
+      arguments:
       - x-message-ttl: 8640000
       - x-expires: 8640000
       - x-dead-letter-exchange: my-new-exchange
+      auto_delete: false
+      durable: true
+      passwd: password
+      user: saltstack
+      vhost: test_vhost
   remove_guest_user: true
   retry_option:
     attempts: 0