Update Helm Charts to Support Multi Node Etcd Cluster (#813)

* update helm charts

* Support Multi-node etcd cluster including peer TLS

* Add OCS store configuration to charts

* update docs to note dynamic cluster configuration is not supported with helm charts

chart/etcd-backup-restore/templates/etcd-backup-secret.yaml

@@ -14,6 +14,10 @@ data:
   region: {{ .Values.backup.s3.region | b64enc }}
   secretAccessKey: {{ .Values.backup.s3.secretAccessKey | b64enc  }}
   accessKeyID: {{ .Values.backup.s3.accessKeyID | b64enc }}
+  s3ForcePathStyle: {{ .Values.backup.s3.s3ForcePathStyle | b64enc}}
+  {{- if .Values.backup.s3.endpoint }}
+  endpoint: {{ .Values.backup.s3.endpoint | b64enc }}
+  {{- end }}
 {{- else if eq .Values.backup.storageProvider "ABS" }}
   storageAccount: {{ .Values.backup.abs.storageAccount | b64enc }}
   storageKey : {{ .Values.backup.abs.storageKey | b64enc }}
@@ -42,6 +46,11 @@ data:
   endpoint: {{ .Values.backup.oss.endpoint | b64enc }}
   accessKeySecret: {{ .Values.backup.oss.accessKeySecret | b64enc }}
   accessKeyID: {{ .Values.backup.oss.accessKeyID | b64enc }}
+{{- else if eq .Values.backup.storageProvider "OCS"}}
+  accessKeyID: {{ .Values.backup.ocs.accessKeyID | b64enc }}
+  secretAccessKey: {{ .Values.backup.ocs.secretAccessKey | b64enc }}
+  endpoint: {{ .Values.backup.ocs.endpoint | b64enc }}
+  region: {{ .Values.backup.ocs.region | b64enc }}
 {{- else if eq .Values.backup.storageProvider "ECS" }}
   endpoint: {{ .Values.backup.ecs.endpoint | b64enc }}
   accessKeyID: {{ .Values.backup.ecs.accessKeyID | b64enc }}

chart/etcd-backup-restore/templates/etcd-ca-secret.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.etcdTLS }}
+{{- if .Values.tls.etcd }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -10,5 +10,5 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 type: Opaque
 data:
-  ca.crt: {{ .Values.etcdTLS.caBundle | b64enc }}
-{{- end }}
+  bundle.crt: {{ .Values.tls.etcd.ca | b64enc }}
+{{- end }}

chart/etcd-backup-restore/templates/etcd-client-service.yaml

@@ -16,5 +16,14 @@ spec:
   ports:
   - name: client
     protocol: TCP
-    port: {{ .Values.servicePorts.client }}
-    targetPort: {{ .Values.servicePorts.client }}
+    port: {{ .Values.servicePorts.etcd.client }}
+    targetPort: {{ .Values.servicePorts.etcd.client }}
+  - name: peer
+    protocol: TCP
+    port: {{ .Values.servicePorts.etcd.peer }}
+    targetPort: {{ .Values.servicePorts.etcd.peer }}
+  - name: backuprestore
+    protocol: TCP
+    port: {{ .Values.servicePorts.etcdBackupRestore.server }}
+    targetPort: {{ .Values.servicePorts.etcdBackupRestore.server }}
+

chart/etcd-backup-restore/templates/etcd-tls-secret.yaml → chart/etcd-backup-restore/templates/etcd-client-tls-secret.yaml

@@ -1,15 +1,15 @@
-{{- if .Values.etcdTLS }}
+{{- if .Values.tls.etcd }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Release.Name }}-etcd-tls
+  name: {{ .Release.Name }}-etcd-client-tls
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: etcd
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 type: kubernetes.io/tls
 data:
-  tls.crt: {{ .Values.etcdTLS.crt | b64enc }}
-  tls.key: {{ .Values.etcdTLS.key | b64enc }}
-{{- end }}
+  tls.crt: {{ .Values.tls.etcd.client.crt | b64enc }}
+  tls.key: {{ .Values.tls.etcd.client.key | b64enc }}
+{{- end }}

chart/etcd-backup-restore/templates/etcd-configmap.yaml

@@ -10,6 +10,21 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 data:
   etcd.conf.yaml: |-
+    {{- $replicas := int .Values.replicas }}
+    # precompute the peer scheme based on whether or not the peer is tls enabled
+    {{- $peerScheme := "http" }}
+    {{- if .Values.tls.etcd.peer }}
+    {{- $peerScheme = "https" }}
+    {{- end }}
+    # store the root context for later use
+    {{- $root := . }}
+    # store the cluster entries in a list to be used for the initial-cluster configuration
+    {{- $clusterEntries := list }}
+    {{- range $i := until $replicas }}
+    {{- $entry := printf "%s-etcd-%d=%s://%s-etcd-%d.%s-etcd-peer.%s.svc:%d" $root.Release.Name $i $peerScheme $root.Release.Name $i $root.Release.Name $root.Release.Namespace (int $root.Values.servicePorts.etcd.peer) }}
+    {{- $clusterEntries = append $clusterEntries $entry }}
+    {{- end }}
+
     # Human-readable name for this member.
     name: {{ .Release.Name }}-etcd
 
@@ -22,21 +37,41 @@ data:
     # Number of committed transactions to trigger a snapshot to disk.
     snapshot-count: 75000
 
+    enable-v2: false
+
     # Raise alarms when backend size exceeds the given quota. 0 means use the
     # default quota.
     {{- if .Values.backup.etcdQuotaBytes }}
     quota-backend-bytes: {{ int $.Values.backup.etcdQuotaBytes }}
     {{- end }}
 
     # List of comma separated URLs to listen on for client traffic.
-    listen-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
+    listen-client-urls: {{ if .Values.tls.etcd }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.etcd.client }}
+
+    # List of comma separated URLs to listen on for peer traffic.
+    listen-peer-urls: {{ $peerScheme }}://0.0.0.0:{{ .Values.servicePorts.etcd.peer }}
+
+    # List of each member's client URLs to advertise to the public.
+    # Each member should include it's client URLs under the member name.
+    advertise-client-urls:
+      {{- range $i := until $replicas }}
+      {{ $root.Release.Name }}-etcd-{{ $i }}:
+      - {{ if $root.Values.tls.etcd }}https{{ else }}http{{ end }}://{{ $root.Release.Name }}-etcd-{{ $i }}.{{ $root.Release.Name }}-etcd-peer.{{ $root.Release.Namespace }}.svc:{{ $root.Values.servicePorts.etcd.client }}
+      {{- end }}
+
+    # List of each member's peer URLs to advertise to the public
+    # Each member should include it's peer URLs under the member name.
+    initial-advertise-peer-urls:
+      {{- range $i := until $replicas }}
+      {{ $root.Release.Name }}-etcd-{{ $i }}:
+      - {{ $peerScheme }}://{{ $root.Release.Name }}-etcd-{{ $i }}.{{ $root.Release.Name }}-etcd-peer.{{ $root.Release.Namespace }}.svc:{{ $root.Values.servicePorts.etcd.peer }}
+      {{- end }}
 
-    # List of this member's client URLs to advertise to the public.
-    # The URLs needed to be a comma-separated list.
-    advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }}
+    # List of server endpoints with which this cluster should be started
+    initial-cluster: {{ join "," $clusterEntries }}
 
     # Initial cluster token for the etcd cluster during bootstrap.
-    initial-cluster-token: 'new'
+    initial-cluster-token: 'etcd-cluster'
 
     # Initial cluster state ('new' or 'existing').
     initial-cluster-state: 'new'
@@ -53,17 +88,35 @@ data:
     {{- end }}
     {{- end }}
 
-{{- if .Values.etcdTLS }}
+{{- if .Values.tls.etcd }}
     client-transport-security:
-      # Path to the client server TLS cert file.
-      cert-file: /var/etcd/ssl/tls/tls.crt
+      # Path to the etcd server TLS cert file.
+      cert-file: /var/etcd/ssl/server/tls.crt
 
-      # Path to the client server TLS key file.
-      key-file: /var/etcd/ssl/tls/tls.key
+      # Path to the etcd server TLS key file.
+      key-file: /var/etcd/ssl/server/tls.key
 
       # Enable client cert authentication.
       client-cert-auth: true
 
-      # Path to the client server TLS trusted CA cert file.
-      trusted-ca-file: /var/etcd/ssl/ca/ca.crt
+      # Path to the etcd server TLS trusted CA cert file.
+      trusted-ca-file: /var/etcd/ssl/ca/bundle.crt
+
+      auto-tls: false
+  {{- if .Values.tls.etcd.peer }}
+    peer-transport-security:
+      # Path to the etcd peer server TLS cert file.
+      cert-file: /var/etcd/ssl/peer/server/tls.crt
+
+      # Path to the etcd peer server TLS key file.
+      key-file: /var/etcd/ssl/peer/server/tls.key
+
+      # Enable peer client cert authentication.
+      client-cert-auth: true
+
+      # Path to the etcd peer server TLS trusted CA cert file.
+      trusted-ca-file: /var/etcd/ssl/peer/ca/bundle.crt
+
+      auto-tls: false
+  {{- end }}
 {{- end }}

chart/etcd-backup-restore/templates/etcd-peer-ca-secret.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.tls.etcd.peer }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-etcd-peer-ca
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: etcd
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+type: Opaque
+data:
+  bundle.crt: {{ .Values.tls.etcd.peer.ca | b64enc }}
+{{- end }}

chart/etcd-backup-restore/templates/etcd-peer-server-tls-secret.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.tls.etcd.peer }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-etcd-peer-server-tls
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: etcd
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ .Values.tls.etcd.peer.server.crt | b64enc }}
+  tls.key: {{ .Values.tls.etcd.peer.server.key | b64enc }}
+{{- end }}

chart/etcd-backup-restore/templates/backup-client-service.yaml → chart/etcd-backup-restore/templates/etcd-peer-service.yaml

@@ -1,20 +1,27 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Release.Name }}-backup-client
+  name: {{ .Release.Name }}-etcd-peer
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: etcd
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
+  publishNotReadyAddresses: true
   type: ClusterIP
+  clusterIP: None
+  clusterIPs:
+    - None
+  internalTrafficPolicy: Cluster
+  ipFamilyPolicy: SingleStack
   sessionAffinity: None
   selector:
     app.kubernetes.io/name: etcd
     app.kubernetes.io/instance: {{ .Release.Name }}
   ports:
-  - name: client
+  - name: peer
     protocol: TCP
-    port: {{ .Values.servicePorts.backupRestore }}
-    targetPort: {{ .Values.servicePorts.backupRestore }}
+    port: {{ .Values.servicePorts.etcd.peer }}
+    targetPort: {{ .Values.servicePorts.etcd.peer }}
+

chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml → chart/etcd-backup-restore/templates/etcd-server-tls-secret.yaml

@@ -1,15 +1,15 @@
-{{- if .Values.backupRestoreTLS }}
+{{- if .Values.tls.etcd }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Release.Name }}-etcdbr-tls
+  name: {{ .Release.Name }}-etcd-server-tls
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: etcd
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 type: kubernetes.io/tls
 data:
-  tls.crt: {{ .Values.backupRestoreTLS.crt | b64enc }}
-  tls.key: {{ .Values.backupRestoreTLS.key | b64enc }}
-{{- end }}
+  tls.crt: {{ .Values.tls.etcd.server.crt | b64enc }}
+  tls.key: {{ .Values.tls.etcd.server.key | b64enc }}
+{{- end }}