agentless: enable admin partitions with agentless

* integrate partition-init job with consul-server-connection-manager * integrate controller with consul-server-connection-manager * modify partition token to be be able read all services so that it works with connection manager
hashicorp · Sep 27, 2022 · 76bce3a · 76bce3a
1 parent 2ea0255
commit 76bce3a
Show file tree

Hide file tree

Showing 37 changed files with 793 additions and 1,191 deletions.
diff --git a/acceptance/tests/partitions/partitions_connect_test.go b/acceptance/tests/partitions/partitions_connect_test.go
diff --git a/charts/consul/templates/controller-deployment.yaml b/charts/consul/templates/controller-deployment.yaml
@@ -1,5 +1,6 @@
 {{- if .Values.controller.enabled }}
 {{- if and .Values.global.adminPartitions.enabled (not .Values.global.enableConsulNamespaces) }}{{ fail "global.enableConsulNamespaces must be true if global.adminPartitions.enabled=true" }}{{ end }}
+{{- if and .Values.externalServers.enabled (not .Values.externalServers.hosts) }}{{ fail "externalServers.hosts must be set if externalServers.enabled is true" }}{{ end -}}
 {{ template "consul.validateVaultWebhookCertConfiguration" . }}
 apiVersion: apps/v1
 kind: Deployment
@@ -65,107 +66,21 @@ spec:
         {{- end }}
         {{- end }}
     spec:
-      {{- if .Values.global.acls.manageSystemACLs }}
-      initContainers:
-      - name: controller-acl-init
-        env:
-          {{- if not .Values.externalServers.enabled }}
-          - name: CONSUL_HTTP_ADDR
-            {{- if .Values.global.tls.enabled }}
-            value: https://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8501
-            {{- else }}
-            value: http://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8500
-            {{- end }}
-          {{- end }}
-          {{- if (and .Values.global.tls.enabled (not .Values.externalServers.useSystemRoots)) }}
-          - name: CONSUL_CACERT
-            {{- if .Values.global.secretsBackend.vault.enabled }}
-            value: "/vault/secrets/serverca.crt"
-            {{- else }}
-            value: "/consul/tls/ca/tls.crt"
-            {{- end }}
-          {{- end }}
-        image: {{ .Values.global.imageK8S }}
-        volumeMounts:
-        - mountPath: /consul/login
-          name: consul-data
-          readOnly: false
-       {{- if and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))}}
-        - name: consul-ca-cert
-          mountPath: /consul/tls/ca
-          readOnly: true
-        {{- end }}
-        command:
-          - "/bin/sh"
-          - "-ec"
-          - |
-            consul-k8s-control-plane acl-init \
-              -component-name=controller \
-              {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
-              -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \
-              -primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \
-              {{- else }}
-              -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \
-              {{- end }}
-              {{- if .Values.global.adminPartitions.enabled }}
-              -partition={{ .Values.global.adminPartitions.name }} \
-              {{- end }}
-              {{- if .Values.externalServers.enabled }}
-              {{- if .Values.global.tls.enabled }}
-              -use-https \
-              {{- end }}
-              {{- range .Values.externalServers.hosts }}
-              -server-address={{ quote . }} \
-              {{- end }}
-              -server-port={{ .Values.externalServers.httpsPort }} \
-              {{- if .Values.externalServers.tlsServerName }}
-              -tls-server-name={{ .Values.externalServers.tlsServerName }} \
-              {{- end }}
-              {{- end }}
-              -consul-api-timeout={{ .Values.global.consulAPITimeout }} \
-              -log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
-              -log-json={{ .Values.global.logJSON }}
-        resources:
-          requests:
-            memory: "25Mi"
-            cpu: "50m"
-          limits:
-            memory: "25Mi"
-            cpu: "50m"
-      {{- end }}
       containers:
       - command:
         - "/bin/sh"
         - "-ec"
         - |
           consul-k8s-control-plane controller \
-            -consul-api-timeout={{ .Values.global.consulAPITimeout }} \
             -log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
             -log-json={{ .Values.global.logJSON }} \
             -resource-prefix={{ template "consul.fullname" . }} \
-            {{- if and .Values.externalServers.enabled (not .Values.externalServers.hosts) }}{{ fail "externalServers.hosts must be set if externalServers.enabled is true" }}{{ end -}}
-            {{- if .Values.externalServers.enabled }}
-            {{- if .Values.global.tls.enabled }}
-            -use-https \
-            {{- end }}
-            {{- range .Values.externalServers.hosts }}
-            -server-address={{ quote . }} \
-            {{- end }}
-            -server-port={{ .Values.externalServers.httpsPort }} \
-            {{- if .Values.externalServers.tlsServerName }}
-            -tls-server-name={{ .Values.externalServers.tlsServerName }} \
-            {{- end }}
-            {{- end }}
             {{- if and .Values.global.secretsBackend.vault.enabled .Values.global.secretsBackend.vault.controller.tlsCert.secretName }}
             -enable-webhook-ca-update \
             -webhook-tls-cert-dir=/vault/secrets/controller-webhook/certs \
             {{- else }}
             -webhook-tls-cert-dir=/tmp/controller-webhook/certs \
             {{- end }}
-            -datacenter={{ .Values.global.datacenter }} \
-            {{- if .Values.global.adminPartitions.enabled }}
-            -partition={{ .Values.global.adminPartitions.name }} \
-            {{- end }}
             -enable-leader-election \
             {{- if .Values.global.enableConsulNamespaces }}
             -enable-namespaces=true \
@@ -182,44 +97,39 @@ spec:
             -consul-cross-namespace-acl-policy=cross-namespace-policy \
             {{- end }}
             {{- end }}
-        {{- if .Values.global.acls.manageSystemACLs }}
-        lifecycle:
-          preStop:
-            exec:
-              command:
-                - "/bin/sh"
-                - "-ec"
-                - |
-                  consul-k8s-control-plane consul-logout -consul-api-timeout={{ .Values.global.consulAPITimeout }}
-        {{- end }}
         env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
         {{- if .Values.global.acls.manageSystemACLs }}
-        - name: CONSUL_HTTP_TOKEN_FILE
-          value: "/consul/login/acl-token"
+        - name: CONSUL_LOGIN_AUTH_METHOD
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
+          {{- else }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method
+          {{- end }}
+        - name: CONSUL_LOGIN_DATACENTER
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
+          value: {{ .Values.global.federation.primaryDatacenter }}
+          {{- else }}
+          value: {{ .Values.global.datacenter }}
+          {{- end }}
+        - name: CONSUL_LOGIN_META
+          value: "component=controller,pod=$(POD_NAME)"
         {{- end }}
         {{- if (and .Values.controller.aclToken.secretName .Values.controller.aclToken.secretKey) }}
-        - name: CONSUL_HTTP_TOKEN
+        - name: CONSUL_ACL_TOKEN
           valueFrom:
             secretKeyRef:
               name: {{ .Values.controller.aclToken.secretName }}
               key: {{ .Values.controller.aclToken.secretKey }}
         {{- end }}
-        {{- if not .Values.externalServers.enabled }}
-        - name: CONSUL_HTTP_ADDR
-          {{- if .Values.global.tls.enabled }}
-          value: https://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8501
-          {{- else }}
-          value: http://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8500
-          {{- end }}
-        {{- end }}
-        {{- if (and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))) }}
-        - name: CONSUL_CACERT
-          {{- if .Values.global.secretsBackend.vault.enabled }}
-          value: "/vault/secrets/serverca.crt"
-          {{- else }}
-          value: "/consul/tls/ca/tls.crt"
-          {{- end }}
-        {{- end }}
         image: {{ .Values.global.imageK8S }}
         name: controller
         ports:
@@ -230,10 +140,21 @@ spec:
         resources:
           {{- toYaml . | nindent 12 }}
         {{- end }}
+        startupProbe:
+          tcpSocket:
+            port: 9443
+          initialDelaySeconds: 30
+          failureThreshold: 15
+          periodSeconds: 2
+          timeoutSeconds: 5
+        readinessProbe:
+          tcpSocket:
+            port: 9443
+          failureThreshold: 2
+          initialDelaySeconds: 2
+          successThreshold: 1
+          timeoutSeconds: 5
         volumeMounts:
-        - mountPath: /consul/login
-          name: consul-data
-          readOnly: true
         {{- if not (and .Values.global.secretsBackend.vault.enabled .Values.global.secretsBackend.vault.controller.tlsCert.secretName) }}
         - mountPath: /tmp/controller-webhook/certs
           name: cert
@@ -266,9 +187,6 @@ spec:
               path: tls.crt
       {{- end }}
       {{- end }}
-      - name: consul-data
-        emptyDir:
-          medium: "Memory"
       serviceAccountName: {{ template "consul.fullname" . }}-controller
       {{- if .Values.controller.nodeSelector }}
       nodeSelector:

diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml
@@ -152,7 +152,7 @@ spec:
         volumeMounts:
         - name: consul-service
           mountPath: /consul/service
-        {{- if .Values.global.tls.enabled }}
+        {{- if and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) }}
         - name: consul-ca-cert
           mountPath: /consul/tls/ca
           readOnly: true
@@ -175,7 +175,7 @@ spec:
         - mountPath: /consul/service
           name: consul-service
           readOnly: true
-        {{- if .Values.global.tls.enabled }}
+        {{- if and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) }}
         - name: consul-ca-cert
           mountPath: /consul/tls/ca
           readOnly: true
@@ -195,7 +195,7 @@ spec:
         - |
           consul-dataplane \
             {{- if .Values.externalServers.enabled }}
-            -addresses={{ .Values.externalServers.hosts | first }} \
+            -addresses={{ .Values.externalServers.hosts | first | quote }} \
             {{- else }}
             -addresses="{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc" \
             {{- end }}

diff --git a/charts/consul/templates/partition-init-job.yaml b/charts/consul/templates/partition-init-job.yaml
@@ -3,6 +3,7 @@
 {{- template "consul.reservedNamesFailer" (list .Values.global.adminPartitions.name "global.adminPartitions.name") }}
 {{- if and (not .Values.externalServers.enabled) (ne .Values.global.adminPartitions.name "default") }}{{ fail "externalServers.enabled needs to be true and configured to create a non-default partition." }}{{ end -}}
 {{- if and .Values.global.secretsBackend.vault.enabled .Values.global.acls.manageSystemACLs (not .Values.global.secretsBackend.vault.adminPartitionsRole) }}{{ fail "global.secretsBackend.vault.adminPartitionsRole is required when global.secretsBackend.vault.enabled and global.acls.manageSystemACLs are true." }}{{ end -}}
+{{- if and .Values.externalServers.enabled (not .Values.externalServers.hosts) }}{{ fail "externalServers.hosts must be set if externalServers.enabled is true" }}{{ end -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -75,22 +76,19 @@ spec:
         - name: partition-init-job
           image: {{ .Values.global.imageK8S }}
           env:
-            - name: NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
-            {{- if .Values.global.secretsBackend.vault.enabled }}
-            - name: CONSUL_HTTP_TOKEN_FILE
-              value: /vault/secrets/bootstrap-token
-            {{- else }}
-            - name: CONSUL_HTTP_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.global.acls.bootstrapToken.secretName }}
-                  key: {{ .Values.global.acls.bootstrapToken.secretKey }}
-            {{- end }}
-            {{- end }}
+          {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 10 }}
+          {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+          {{- if .Values.global.secretsBackend.vault.enabled }}
+          - name: CONSUL_ACL_TOKEN_FILE
+            value: /vault/secrets/bootstrap-token
+          {{- else }}
+          - name: CONSUL_ACL_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.global.acls.bootstrapToken.secretName }}
+                key: {{ .Values.global.acls.bootstrapToken.secretKey }}
+          {{- end }}
+          {{- end }}
           {{- if .Values.global.tls.enabled  }}
           {{- if not (or .Values.externalServers.useSystemRoots .Values.global.secretsBackend.vault.enabled) }}
           volumeMounts:
@@ -104,30 +102,8 @@ spec:
             - "-ec"
             - |
               consul-k8s-control-plane partition-init \
-                -consul-api-timeout={{ .Values.global.consulAPITimeout }} \
                 -log-level={{ .Values.global.logLevel }} \
                 -log-json={{ .Values.global.logJSON }} \
-
-                {{- if and .Values.externalServers.enabled (not .Values.externalServers.hosts) }}{{ fail "externalServers.hosts must be set if externalServers.enabled is true" }}{{ end -}}
-                {{- range .Values.externalServers.hosts }}
-                -server-address={{ quote . }} \
-                {{- end }}
-                -server-port={{ .Values.externalServers.httpsPort }} \
-
-                {{- if .Values.global.tls.enabled }}
-                -use-https \
-                {{- if not .Values.externalServers.useSystemRoots }}
-                {{- if .Values.global.secretsBackend.vault.enabled }}
-                -ca-file=/vault/secrets/serverca.crt \
-                {{- else }}
-                -ca-file=/consul/tls/ca/tls.crt \
-                {{- end }}
-                {{- end }}
-                {{- if .Values.externalServers.tlsServerName }}
-                -tls-server-name={{ .Values.externalServers.tlsServerName }} \
-                {{- end }}
-                {{- end }}
-                -partition-name={{ .Values.global.adminPartitions.name }}
           resources:
             requests:
               memory: "50Mi"

diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml
@@ -80,6 +80,7 @@ spec:
       {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName .Values.global.acls.bootstrapToken.secretName) }}
       volumes:
         {{- if and .Values.global.tls.enabled (not .Values.global.secretsBackend.vault.enabled) }}
+        {{- if not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) }}
         - name: consul-ca-cert
           secret:
             {{- if .Values.global.tls.caCert.secretName }}
@@ -91,6 +92,7 @@ spec:
               - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
                 path: tls.crt
         {{- end }}
+        {{- end }}
         {{- if (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.secretsBackend.vault.enabled)) }}
         - name: bootstrap-token
           secret:
@@ -118,10 +120,12 @@ spec:
           {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName .Values.global.acls.bootstrapToken.secretName) }}
           volumeMounts:
             {{- if and .Values.global.tls.enabled (not .Values.global.secretsBackend.vault.enabled) }}
+            {{- if not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) }}
             - name: consul-ca-cert
               mountPath: /consul/tls/ca
               readOnly: true
             {{- end }}
+            {{- end }}
             {{- if (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.secretsBackend.vault.enabled)) }}
             - name: bootstrap-token
               mountPath: /consul/acl/tokens