From 325269cb95ebf7f2ad01095b73318f72463c024d Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Wed, 15 Jan 2025 22:08:36 +0530 Subject: [PATCH] Update Helm Charts to Support Multi Node Etcd Cluster (#813) * update helm charts * Support Multi-node etcd cluster including peer TLS * Add OCS store configuration to charts * update docs to note dynamic cluster configuration is not supported with helm charts --- .../templates/etcd-backup-secret.yaml | 9 + .../templates/etcd-ca-secret.yaml | 6 +- .../templates/etcd-client-service.yaml | 13 +- ...ecret.yaml => etcd-client-tls-secret.yaml} | 10 +- .../templates/etcd-configmap.yaml | 77 +++- .../templates/etcd-peer-ca-secret.yaml | 14 + .../etcd-peer-server-tls-secret.yaml | 15 + ...nt-service.yaml => etcd-peer-service.yaml} | 15 +- ...ecret.yaml => etcd-server-tls-secret.yaml} | 10 +- .../templates/etcd-statefulset.yaml | 408 ++++++++++-------- .../templates/etcdbr-ca-secret.yaml | 6 +- .../templates/etcdbr-client-tls-secret.yaml | 15 + .../templates/etcdbr-server-tls-secret.yaml | 15 + chart/etcd-backup-restore/templates/role.yaml | 18 + .../templates/rolebinding.yaml | 19 + .../templates/serviceaccount.yaml | 11 + chart/etcd-backup-restore/values.yaml | 142 ++++-- docs/deployment/getting_started.md | 2 + 18 files changed, 540 insertions(+), 265 deletions(-) rename chart/etcd-backup-restore/templates/{etcd-tls-secret.yaml => etcd-client-tls-secret.yaml} (57%) create mode 100644 chart/etcd-backup-restore/templates/etcd-peer-ca-secret.yaml create mode 100644 chart/etcd-backup-restore/templates/etcd-peer-server-tls-secret.yaml rename chart/etcd-backup-restore/templates/{backup-client-service.yaml => etcd-peer-service.yaml} (57%) rename chart/etcd-backup-restore/templates/{etcdbr-tls-secret.yaml => etcd-server-tls-secret.yaml} (57%) create mode 100644 chart/etcd-backup-restore/templates/etcdbr-client-tls-secret.yaml create mode 100644 chart/etcd-backup-restore/templates/etcdbr-server-tls-secret.yaml create mode 100644 chart/etcd-backup-restore/templates/role.yaml create mode 100644 chart/etcd-backup-restore/templates/rolebinding.yaml create mode 100644 chart/etcd-backup-restore/templates/serviceaccount.yaml diff --git a/chart/etcd-backup-restore/templates/etcd-backup-secret.yaml b/chart/etcd-backup-restore/templates/etcd-backup-secret.yaml index 0ae067c8e..4fecc4050 100644 --- a/chart/etcd-backup-restore/templates/etcd-backup-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcd-backup-secret.yaml @@ -14,6 +14,10 @@ data: region: {{ .Values.backup.s3.region | b64enc }} secretAccessKey: {{ .Values.backup.s3.secretAccessKey | b64enc }} accessKeyID: {{ .Values.backup.s3.accessKeyID | b64enc }} + s3ForcePathStyle: {{ .Values.backup.s3.s3ForcePathStyle | b64enc}} + {{- if .Values.backup.s3.endpoint }} + endpoint: {{ .Values.backup.s3.endpoint | b64enc }} + {{- end }} {{- else if eq .Values.backup.storageProvider "ABS" }} storageAccount: {{ .Values.backup.abs.storageAccount | b64enc }} storageKey : {{ .Values.backup.abs.storageKey | b64enc }} @@ -42,6 +46,11 @@ data: endpoint: {{ .Values.backup.oss.endpoint | b64enc }} accessKeySecret: {{ .Values.backup.oss.accessKeySecret | b64enc }} accessKeyID: {{ .Values.backup.oss.accessKeyID | b64enc }} +{{- else if eq .Values.backup.storageProvider "OCS"}} + accessKeyID: {{ .Values.backup.ocs.accessKeyID | b64enc }} + secretAccessKey: {{ .Values.backup.ocs.secretAccessKey | b64enc }} + endpoint: {{ .Values.backup.ocs.endpoint | b64enc }} + region: {{ .Values.backup.ocs.region | b64enc }} {{- else if eq .Values.backup.storageProvider "ECS" }} endpoint: {{ .Values.backup.ecs.endpoint | b64enc }} accessKeyID: {{ .Values.backup.ecs.accessKeyID | b64enc }} diff --git a/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml b/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml index 7a4abc01c..7edd0b117 100644 --- a/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml @@ -1,4 +1,4 @@ -{{- if .Values.etcdTLS }} +{{- if .Values.tls.etcd }} apiVersion: v1 kind: Secret metadata: @@ -10,5 +10,5 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} type: Opaque data: - ca.crt: {{ .Values.etcdTLS.caBundle | b64enc }} -{{- end }} \ No newline at end of file + bundle.crt: {{ .Values.tls.etcd.ca | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/etcd-client-service.yaml b/chart/etcd-backup-restore/templates/etcd-client-service.yaml index 4ce78b2f7..13ef1f559 100644 --- a/chart/etcd-backup-restore/templates/etcd-client-service.yaml +++ b/chart/etcd-backup-restore/templates/etcd-client-service.yaml @@ -16,5 +16,14 @@ spec: ports: - name: client protocol: TCP - port: {{ .Values.servicePorts.client }} - targetPort: {{ .Values.servicePorts.client }} + port: {{ .Values.servicePorts.etcd.client }} + targetPort: {{ .Values.servicePorts.etcd.client }} + - name: peer + protocol: TCP + port: {{ .Values.servicePorts.etcd.peer }} + targetPort: {{ .Values.servicePorts.etcd.peer }} + - name: backuprestore + protocol: TCP + port: {{ .Values.servicePorts.etcdBackupRestore.server }} + targetPort: {{ .Values.servicePorts.etcdBackupRestore.server }} + \ No newline at end of file diff --git a/chart/etcd-backup-restore/templates/etcd-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcd-client-tls-secret.yaml similarity index 57% rename from chart/etcd-backup-restore/templates/etcd-tls-secret.yaml rename to chart/etcd-backup-restore/templates/etcd-client-tls-secret.yaml index b5bfc3239..fbf759a51 100644 --- a/chart/etcd-backup-restore/templates/etcd-tls-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcd-client-tls-secret.yaml @@ -1,8 +1,8 @@ -{{- if .Values.etcdTLS }} +{{- if .Values.tls.etcd }} apiVersion: v1 kind: Secret metadata: - name: {{ .Release.Name }}-etcd-tls + name: {{ .Release.Name }}-etcd-client-tls namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: etcd @@ -10,6 +10,6 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} type: kubernetes.io/tls data: - tls.crt: {{ .Values.etcdTLS.crt | b64enc }} - tls.key: {{ .Values.etcdTLS.key | b64enc }} -{{- end }} \ No newline at end of file + tls.crt: {{ .Values.tls.etcd.client.crt | b64enc }} + tls.key: {{ .Values.tls.etcd.client.key | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/etcd-configmap.yaml b/chart/etcd-backup-restore/templates/etcd-configmap.yaml index 68fcfb7cc..c21680d3c 100644 --- a/chart/etcd-backup-restore/templates/etcd-configmap.yaml +++ b/chart/etcd-backup-restore/templates/etcd-configmap.yaml @@ -10,6 +10,21 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} data: etcd.conf.yaml: |- + {{- $replicas := int .Values.replicas }} + # precompute the peer scheme based on whether or not the peer is tls enabled + {{- $peerScheme := "http" }} + {{- if .Values.tls.etcd.peer }} + {{- $peerScheme = "https" }} + {{- end }} + # store the root context for later use + {{- $root := . }} + # store the cluster entries in a list to be used for the initial-cluster configuration + {{- $clusterEntries := list }} + {{- range $i := until $replicas }} + {{- $entry := printf "%s-etcd-%d=%s://%s-etcd-%d.%s-etcd-peer.%s.svc:%d" $root.Release.Name $i $peerScheme $root.Release.Name $i $root.Release.Name $root.Release.Namespace (int $root.Values.servicePorts.etcd.peer) }} + {{- $clusterEntries = append $clusterEntries $entry }} + {{- end }} + # Human-readable name for this member. name: {{ .Release.Name }}-etcd @@ -22,6 +37,8 @@ data: # Number of committed transactions to trigger a snapshot to disk. snapshot-count: 75000 + enable-v2: false + # Raise alarms when backend size exceeds the given quota. 0 means use the # default quota. {{- if .Values.backup.etcdQuotaBytes }} @@ -29,14 +46,32 @@ data: {{- end }} # List of comma separated URLs to listen on for client traffic. - listen-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }} + listen-client-urls: {{ if .Values.tls.etcd }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.etcd.client }} + + # List of comma separated URLs to listen on for peer traffic. + listen-peer-urls: {{ $peerScheme }}://0.0.0.0:{{ .Values.servicePorts.etcd.peer }} + + # List of each member's client URLs to advertise to the public. + # Each member should include it's client URLs under the member name. + advertise-client-urls: + {{- range $i := until $replicas }} + {{ $root.Release.Name }}-etcd-{{ $i }}: + - {{ if $root.Values.tls.etcd }}https{{ else }}http{{ end }}://{{ $root.Release.Name }}-etcd-{{ $i }}.{{ $root.Release.Name }}-etcd-peer.{{ $root.Release.Namespace }}.svc:{{ $root.Values.servicePorts.etcd.client }} + {{- end }} + + # List of each member's peer URLs to advertise to the public + # Each member should include it's peer URLs under the member name. + initial-advertise-peer-urls: + {{- range $i := until $replicas }} + {{ $root.Release.Name }}-etcd-{{ $i }}: + - {{ $peerScheme }}://{{ $root.Release.Name }}-etcd-{{ $i }}.{{ $root.Release.Name }}-etcd-peer.{{ $root.Release.Namespace }}.svc:{{ $root.Values.servicePorts.etcd.peer }} + {{- end }} - # List of this member's client URLs to advertise to the public. - # The URLs needed to be a comma-separated list. - advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }} + # List of server endpoints with which this cluster should be started + initial-cluster: {{ join "," $clusterEntries }} # Initial cluster token for the etcd cluster during bootstrap. - initial-cluster-token: 'new' + initial-cluster-token: 'etcd-cluster' # Initial cluster state ('new' or 'existing'). initial-cluster-state: 'new' @@ -53,17 +88,35 @@ data: {{- end }} {{- end }} -{{- if .Values.etcdTLS }} +{{- if .Values.tls.etcd }} client-transport-security: - # Path to the client server TLS cert file. - cert-file: /var/etcd/ssl/tls/tls.crt + # Path to the etcd server TLS cert file. + cert-file: /var/etcd/ssl/server/tls.crt - # Path to the client server TLS key file. - key-file: /var/etcd/ssl/tls/tls.key + # Path to the etcd server TLS key file. + key-file: /var/etcd/ssl/server/tls.key # Enable client cert authentication. client-cert-auth: true - # Path to the client server TLS trusted CA cert file. - trusted-ca-file: /var/etcd/ssl/ca/ca.crt + # Path to the etcd server TLS trusted CA cert file. + trusted-ca-file: /var/etcd/ssl/ca/bundle.crt + + auto-tls: false + {{- if .Values.tls.etcd.peer }} + peer-transport-security: + # Path to the etcd peer server TLS cert file. + cert-file: /var/etcd/ssl/peer/server/tls.crt + + # Path to the etcd peer server TLS key file. + key-file: /var/etcd/ssl/peer/server/tls.key + + # Enable peer client cert authentication. + client-cert-auth: true + + # Path to the etcd peer server TLS trusted CA cert file. + trusted-ca-file: /var/etcd/ssl/peer/ca/bundle.crt + + auto-tls: false + {{- end }} {{- end }} diff --git a/chart/etcd-backup-restore/templates/etcd-peer-ca-secret.yaml b/chart/etcd-backup-restore/templates/etcd-peer-ca-secret.yaml new file mode 100644 index 000000000..e4f549e95 --- /dev/null +++ b/chart/etcd-backup-restore/templates/etcd-peer-ca-secret.yaml @@ -0,0 +1,14 @@ +{{- if .Values.tls.etcd.peer }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-etcd-peer-ca + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +type: Opaque +data: + bundle.crt: {{ .Values.tls.etcd.peer.ca | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/etcd-peer-server-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcd-peer-server-tls-secret.yaml new file mode 100644 index 000000000..8bb5d505e --- /dev/null +++ b/chart/etcd-backup-restore/templates/etcd-peer-server-tls-secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.tls.etcd.peer }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-etcd-peer-server-tls + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.tls.etcd.peer.server.crt | b64enc }} + tls.key: {{ .Values.tls.etcd.peer.server.key | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/backup-client-service.yaml b/chart/etcd-backup-restore/templates/etcd-peer-service.yaml similarity index 57% rename from chart/etcd-backup-restore/templates/backup-client-service.yaml rename to chart/etcd-backup-restore/templates/etcd-peer-service.yaml index ed73f9b47..941b2701d 100644 --- a/chart/etcd-backup-restore/templates/backup-client-service.yaml +++ b/chart/etcd-backup-restore/templates/etcd-peer-service.yaml @@ -1,20 +1,27 @@ apiVersion: v1 kind: Service metadata: - name: {{ .Release.Name }}-backup-client + name: {{ .Release.Name }}-etcd-peer namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: etcd app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} spec: + publishNotReadyAddresses: true type: ClusterIP + clusterIP: None + clusterIPs: + - None + internalTrafficPolicy: Cluster + ipFamilyPolicy: SingleStack sessionAffinity: None selector: app.kubernetes.io/name: etcd app.kubernetes.io/instance: {{ .Release.Name }} ports: - - name: client + - name: peer protocol: TCP - port: {{ .Values.servicePorts.backupRestore }} - targetPort: {{ .Values.servicePorts.backupRestore }} + port: {{ .Values.servicePorts.etcd.peer }} + targetPort: {{ .Values.servicePorts.etcd.peer }} + \ No newline at end of file diff --git a/chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcd-server-tls-secret.yaml similarity index 57% rename from chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml rename to chart/etcd-backup-restore/templates/etcd-server-tls-secret.yaml index ee8ddf419..f9b166dd2 100644 --- a/chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcd-server-tls-secret.yaml @@ -1,8 +1,8 @@ -{{- if .Values.backupRestoreTLS }} +{{- if .Values.tls.etcd }} apiVersion: v1 kind: Secret metadata: - name: {{ .Release.Name }}-etcdbr-tls + name: {{ .Release.Name }}-etcd-server-tls namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: etcd @@ -10,6 +10,6 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} type: kubernetes.io/tls data: - tls.crt: {{ .Values.backupRestoreTLS.crt | b64enc }} - tls.key: {{ .Values.backupRestoreTLS.key | b64enc }} -{{- end }} \ No newline at end of file + tls.crt: {{ .Values.tls.etcd.server.crt | b64enc }} + tls.key: {{ .Values.tls.etcd.server.key | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/etcd-statefulset.yaml b/chart/etcd-backup-restore/templates/etcd-statefulset.yaml index ee771be44..b66774ea0 100644 --- a/chart/etcd-backup-restore/templates/etcd-statefulset.yaml +++ b/chart/etcd-backup-restore/templates/etcd-statefulset.yaml @@ -10,8 +10,9 @@ metadata: spec: updateStrategy: type: RollingUpdate - serviceName: {{ .Release.Name }}-etcd-client - replicas: 1 + serviceName: {{ .Release.Name }}-etcd-peer + replicas: {{ (int .Values.replicas) }} + podManagementPolicy: Parallel selector: matchLabels: app.kubernetes.io/name: etcd @@ -27,221 +28,212 @@ spec: app.kubernetes.io/name: etcd app.kubernetes.io/instance: {{ .Release.Name }} spec: - containers: - - name: etcd - image: {{ .Values.images.etcd.repository }}:{{ .Values.images.etcd.tag }} - imagePullPolicy: {{ .Values.images.etcd.pullPolicy }} +{{- if and .Values.backup.storageProvider (eq .Values.backup.storageProvider "Local") }} + initContainers: + - args: + - chown -R 65532:65532 /home/nonroot/{{ .Values.backup.storageContainer}} command: - - /var/etcd/bin/bootstrap.sh + - sh + - -c + - -- + name: change-backup-bucket-permissions + image: {{ .Values.images.changeBackupBucketPermissions.repository }}:{{ .Values.images.changeBackupBucketPermissions.tag }} + imagePullPolicy: {{ .Values.images.changeBackupBucketPermissions.pullPolicy }} + volumeMounts: + - name: local-backup + mountPath: /home/nonroot/{{ .Values.backup.storageContainer}} + securityContext: + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 +{{- end }} + containers: + - args: + - start-etcd + - --backup-restore-host-port={{ .Release.Name }}-etcd-local:{{ .Values.servicePorts.etcdBackupRestore.server }} + - --etcd-server-name={{ .Release.Name }}-etcd-local +{{- if .Values.tls.etcdBackupRestore }} + - --backup-restore-tls-enabled=true + - --backup-restore-ca-cert-bundle-path=/var/etcdbr/ssl/ca/bundle.crt +{{- else }} + - --backup-restore-tls-enabled=false +{{- end }} +{{- if .Values.tls.etcd }} + - --etcd-client-cert-path=/var/etcd/ssl/client/tls.crt + - --etcd-client-key-path=/var/etcd/ssl/client/tls.key +{{- end }} + name: etcd + image: {{ .Values.images.etcdWrapper.repository }}:{{ .Values.images.etcdWrapper.tag }} + imagePullPolicy: {{ .Values.images.etcdWrapper.pullPolicy }} readinessProbe: httpGet: -{{- if .Values.backupRestoreTLS }} +{{- if .Values.tls.etcdBackupRestore }} scheme: HTTPS +{{- else }} + scheme: HTTP {{- end }} +{{- if eq (int .Values.replicas) 1 }} path: /healthz - port: {{ .Values.servicePorts.backupRestore }} - initialDelaySeconds: 5 - periodSeconds: 5 - livenessProbe: - exec: - command: - - /bin/sh - - -ec - - ETCDCTL_API=3 - - etcdctl -{{ if .Values.etcdTLS }} - - --cert=/var/etcd/ssl/tls/tls.crt - - --key=/var/etcd/ssl/tls/tls.key - - --cacert=/var/etcd/ssl/ca/ca.crt -{{ end }} - - --endpoints={{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://{{ .Release.Name }}-etcd-0:{{ .Values.servicePorts.client }} -{{- if and .Values.etcdAuth.username .Values.etcdAuth.password }} - - --user={{ .Values.etcdAuth.username }}:{{ .Values.etcdAuth.password }} + port: {{ .Values.servicePorts.etcdBackupRestore.server }} +{{- else }} + path: /readyz + port: {{ .Values.servicePorts.etcdWrapper.server }} {{- end }} - - get - - foo initialDelaySeconds: 15 periodSeconds: 5 + failureThreshold: 5 ports: - - containerPort: {{ .Values.servicePorts.server }} - name: server + - containerPort: {{ .Values.servicePorts.etcd.peer }} + name: peer protocol: TCP - - containerPort: {{ .Values.servicePorts.client }} + - containerPort: {{ .Values.servicePorts.etcd.client }} name: client protocol: TCP resources: {{ toYaml .Values.resources.etcd | indent 10 }} - env: - - name: ENABLE_TLS - value: "{{ if .Values.backupRestoreTLS }}true{{ else }}false{{ end }}" - - name: BACKUP_ENDPOINT - value: "http{{ if .Values.backupRestoreTLS }}s{{ end }}://localhost:{{ .Values.servicePorts.backupRestore }}" - - name: FAIL_BELOW_REVISION_PARAMETER - value: "{{ if .Values.backup.failBelowRevision }}&failbelowrevision={{ int $.Values.backup.failBelowRevision }}{{ end }}" volumeMounts: - name: {{ .Release.Name }}-etcd mountPath: /var/etcd/data/ - - name: etcd-config-file - mountPath: /var/etcd/config/ -{{- if .Values.etcdTLS }} - - name: ca-etcd +{{- if .Values.tls.etcd }} + - name: etcd-ca mountPath: /var/etcd/ssl/ca - - name: etcd-tls - mountPath: /var/etcd/ssl/tls + - name: etcd-server-tls + mountPath: /var/etcd/ssl/server + - name: etcd-client-tls + mountPath: /var/etcd/ssl/client + {{ if .Values.tls.etcd.peer }} + - name: etcd-peer-ca + mountPath: /var/etcd/ssl/peer/ca + - name: etcd-peer-server-tls + mountPath: /var/etcd/ssl/peer/server + {{- end }} {{- end }} -{{- if .Values.backupRestoreTLS }} - - name: ca-etcdbr +{{- if .Values.tls.etcdBackupRestore }} + - name: backup-restore-ca mountPath: /var/etcdbr/ssl/ca {{- end }} - name: backup-restore - command: - - etcdbrctl + args: - server +{{- if .Values.backup.storageProvider}} + # Snapstore flags + - --storage-provider={{ .Values.backup.storageProvider }} + - --store-prefix={{ .Release.Name }}-etcd + # Snapshot flags - --schedule={{ .Values.backup.schedule }} -{{- if eq .Values.backup.garbageCollectionPolicy "LimitBased" }} - - --max-backups={{ .Values.backup.maxBackups }} + - --delta-snapshot-period={{ .Values.backup.deltaSnapshotPeriod }} + - --delta-snapshot-memory-limit={{ int $.Values.backup.deltaSnapshotMemoryLimit }} + # GC flags - --garbage-collection-policy={{ .Values.backup.garbageCollectionPolicy }} -{{- end }} + {{- if eq .Values.backup.garbageCollectionPolicy "LimitBased" }} + - --max-backups={{ .Values.backup.maxBackups }} + {{- end }} - --garbage-collection-period={{ .Values.backup.garbageCollectionPeriod }} - - --data-dir=/var/etcd/data/new.etcd - - --storage-provider={{ .Values.backup.storageProvider }} - - --store-prefix={{ .Release.Name }}-etcd -{{- if .Values.backup.etcdQuotaBytes }} - - --embedded-etcd-quota-bytes={{ int $.Values.backup.etcdQuotaBytes }} + # Snapshot compression and timeout flags + {{- if .Values.backup.compression }} + {{- if .Values.backup.compression.enabled }} + - --compress-snapshots={{ .Values.backup.compression.enabled }} + {{- end }} + {{- if .Values.backup.compression.policy }} + - --compression-policy={{ .Values.backup.compression.policy }} + {{- end }} + {{- end }} + - --etcd-snapshot-timeout={{ .Values.backup.etcdSnapshotTimeout }} +{{- end }} + # Defragmentation flags +{{- if .Values.backup.defragmentationSchedule }} + - --defragmentation-schedule={{ .Values.backup.defragmentationSchedule }} {{- end }} -{{- if .Values.etcdTLS }} - - --cert=/var/etcd/ssl/tls/tls.crt - - --key=/var/etcd/ssl/tls/tls.key - - --cacert=/var/etcd/ssl/ca/ca.crt + - --etcd-defrag-timeout={{ .Values.backup.etcdDefragTimeout}} + # Compaction flags +{{- if .Values.autoCompaction }} + {{- if .Values.autoCompaction.mode }} + - --auto-compaction-mode={{ .Values.autoCompaction.mode }} + {{- end }} + {{- if .Values.autoCompaction.retentionLength }} + - --auto-compaction-retention={{ .Values.autoCompaction.retentionLength }} + {{- end }} +{{- end }} + # Client and Backup TLS command line flags +{{- if .Values.tls.etcd }} + - --cacert=/var/etcd/ssl/ca/bundle.crt + - --cert=/var/etcd/ssl/client/tls.crt + - --key=/var/etcd/ssl/client/tls.key - --insecure-transport=false - --insecure-skip-tls-verify=false - - --endpoints=https://{{ .Release.Name }}-etcd-0:{{ .Values.servicePorts.client }} + - --endpoints=https://{{ .Release.Name }}-etcd-local:{{ .Values.servicePorts.etcd.client }} + - --service-endpoints=https://{{ .Release.Name }}-etcd-client:{{ .Values.servicePorts.etcd.client }} {{ else }} - --insecure-transport=true - --insecure-skip-tls-verify=true - - --endpoints=http://{{ .Release.Name }}-etcd-0:{{ .Values.servicePorts.client }} + - --endpoints=http://{{ .Release.Name }}-etcd-local:{{ .Values.servicePorts.etcd.client }} + - --service-endpoints=http://{{ .Release.Name }}-etcd-client:{{ .Values.servicePorts.etcd.client }} {{- end }} -{{- if .Values.backup.defragmentationSchedule }} - - --defragmentation-schedule={{ .Values.backup.defragmentationSchedule }} +{{- if .Values.tls.etcdBackupRestore }} + - --server-cert=/var/etcdbr/ssl/server/tls.crt + - --server-key=/var/etcdbr/ssl/server/tls.key +{{- end }} + # Other flags + - --data-dir=/var/etcd/data/new.etcd + - --restoration-temp-snapshots-dir=/var/etcd/data/restoration.tmp + - --snapstore-temp-directory=/var/etcd/data/temp +{{- if .Values.backup.etcdQuotaBytes }} + - --embedded-etcd-quota-bytes={{ int $.Values.backup.etcdQuotaBytes }} {{- end }} - --etcd-connection-timeout={{ .Values.backup.etcdConnectionTimeout }} - - --etcd-snapshot-timeout={{ .Values.backup.etcdSnapshotTimeout }} - - --etcd-defrag-timeout={{ .Values.backup.etcdDefragTimeout}} - - --delta-snapshot-period={{ .Values.backup.deltaSnapshotPeriod }} - - --delta-snapshot-memory-limit={{ int $.Values.backup.deltaSnapshotMemoryLimit }} + - --etcd-connection-timeout-leader-election={{ .Values.backup.leaderElection.etcdConnectionTimeout }} + - --reelection-period={{ .Values.backup.leaderElection.reelectionPeriod }} + - --use-etcd-wrapper=true {{- if and .Values.etcdAuth.username .Values.etcdAuth.password }} - --etcd-username={{ .Values.etcdAuth.username }} - --etcd-password={{ .Values.etcdAuth.password }} -{{- end }} -{{- if .Values.backupRestoreTLS }} - - --server-cert=/var/etcdbr/ssl/tls/tls.crt - - --server-key=/var/etcdbr/ssl/tls/tls.key -{{- end }} -{{- if .Values.backup.compression }} - {{- if .Values.backup.compression.enabled }} - - --compress-snapshots={{ .Values.backup.compression.enabled }} - {{- end }} - {{- if .Values.backup.compression.policy }} - - --compression-policy={{ .Values.backup.compression.policy }} - {{- end }} -{{- end }} -{{- if .Values.autoCompaction }} - {{- if .Values.autoCompaction.mode }} - - --auto-compaction-mode={{ .Values.autoCompaction.mode }} - {{- end }} - {{- if .Values.autoCompaction.retentionLength }} - - --auto-compaction-retention={{ .Values.autoCompaction.retentionLength }} - {{- end }} {{- end }} image: {{ .Values.images.etcdBackupRestore.repository }}:{{ .Values.images.etcdBackupRestore.tag }} imagePullPolicy: {{ .Values.images.etcdBackupRestore.pullPolicy }} ports: - - containerPort: {{ .Values.servicePorts.backupRestore }} + - containerPort: {{ .Values.servicePorts.etcdBackupRestore.server }} name: server protocol: TCP resources: {{ toYaml .Values.resources.backup | indent 10 }} env: + - name: "POD_NAME" + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: "POD_NAMESPACE" + value: {{ .Release.Namespace }} - name: STORAGE_CONTAINER value: {{ .Values.backup.storageContainer }} -{{- if eq .Values.backup.storageProvider "S3" }} - - name: "AWS_REGION" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "region" - - name: "AWS_SECRET_ACCESS_KEY" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "secretAccessKey" - - name: "AWS_ACCESS_KEY_ID" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "accessKeyID" -{{- else if eq .Values.backup.storageProvider "ABS" }} - - name: "STORAGE_ACCOUNT" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "storageAccount" - - name: "STORAGE_KEY" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "storageKey" -{{- else if eq .Values.backup.storageProvider "GCS" }} - - name: "GOOGLE_APPLICATION_CREDENTIALS" - value: "/root/.gcp/serviceaccount.json" -{{- else if eq .Values.backup.storageProvider "Swift" }} - - name: "OS_AUTH_URL" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "authURL" - - name: "OS_DOMAIN_NAME" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "domainName" - - name: "OS_USERNAME" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "username" - - name: "OS_PASSWORD" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "password" - - name: "OS_TENANT_NAME" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "tenantName" - - name: "OS_REGION_NAME" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "regionName" -{{- else if eq .Values.backup.storageProvider "OSS" }} - - name: "ALICLOUD_ENDPOINT" +{{- if .Values.backup.storageProvider }} + {{- if eq .Values.backup.storageProvider "S3" }} + - name: "AWS_APPLICATION_CREDENTIALS" + value: "/var/etcd-backup" + {{- if .Values.backup.s3.endpoint }} + - name: "AWS_ENDPOINT_URL_S3" valueFrom: secretKeyRef: name: {{ .Release.Name }}-etcd-backup key: "endpoint" - - name: "ALICLOUD_ACCESS_KEY_SECRET" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "accessKeySecret" - - name: "ALICLOUD_ACCESS_KEY_ID" - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-etcd-backup - key: "accessKeyID" -{{- else if eq .Values.backup.storageProvider "ECS" }} + optional: true + {{- end }} + {{- else if eq .Values.backup.storageProvider "ABS" }} + - name: "AZURE_APPLICATION_CREDENTIALS" + value: "/var/etcd-backup" + {{- else if eq .Values.backup.storageProvider "GCS" }} + - name: "GOOGLE_APPLICATION_CREDENTIALS" + value: "/var/.gcp/serviceaccount.json" + {{- else if eq .Values.backup.storageProvider "Swift" }} + - name: "OPENSTACK_APPLICATION_CREDENTIALS" + value: "/var/etcd-backup" + {{- else if eq .Values.backup.storageProvider "OSS" }} + - name: "ALICLOUD_APPLICATION_CREDENTIALS" + value: "/var/etcd-backup" + {{- else if eq .Values.backup.storageProvider "OCS" }} + - name: "OPENSHIFT_APPLICATION_CREDENTIALS" + value: "/var/etcd-backup" + {{- else if eq .Values.backup.storageProvider "ECS" }} - name: "ECS_ENDPOINT" valueFrom: secretKeyRef: @@ -257,21 +249,22 @@ spec: secretKeyRef: name: {{ .Release.Name }}-etcd-backup key: "secretAccessKey" - {{- if .Values.backup.ecs.disableSsl }} + {{- if .Values.backup.ecs.disableSsl }} - name: "ECS_DISABLE_SSL" valueFrom: secretKeyRef: name: {{ .Release.Name }}-etcd-backup key: "disableSsl" optional: true - {{- end }} - {{- if .Values.backup.ecs.insecureSkipVerify }} + {{- end }} + {{- if .Values.backup.ecs.insecureSkipVerify }} - name: "ECS_INSECURE_SKIP_VERIFY" valueFrom: secretKeyRef: name: {{ .Release.Name }}-etcd-backup key: "insecureSkipVerify" optional: true + {{- end }} {{- end }} {{- end }} volumeMounts: @@ -279,50 +272,90 @@ spec: mountPath: /var/etcd/data/ - name: etcd-config-file mountPath: /var/etcd/config/ -{{- if .Values.etcdTLS }} - - name: ca-etcd +{{- if .Values.tls.etcd }} + - name: etcd-ca mountPath: /var/etcd/ssl/ca - - name: etcd-tls - mountPath: /var/etcd/ssl/tls + - name: etcd-client-tls + mountPath: /var/etcd/ssl/client {{- end }} -{{- if .Values.backupRestoreTLS }} - - name: ca-etcdbr - mountPath: /var/etcdbr/ssl/ca - - name: etcdbr-tls - mountPath: /var/etcdbr/ssl/tls +{{- if .Values.tls.etcdBackupRestore }} + - name: backup-restore-server-tls + mountPath: /var/etcdbr/ssl/server {{- end }} -{{- if eq .Values.backup.storageProvider "GCS" }} +{{- if .Values.backup.storageProvider }} + {{- if eq .Values.backup.storageProvider "Local"}} + - name: local-backup + mountPath: /home/nonroot/{{ .Values.backup.storageContainer}} + {{- else if eq .Values.backup.storageProvider "GCS" }} + - name: etcd-backup + mountPath: "/var/.gcp/" + {{- else }} - name: etcd-backup - mountPath: "/root/.gcp/" + mountPath: "/var/etcd-backup/" + {{- end }} +{{- end }} + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + shareProcessNamespace: true + hostAliases: + - hostnames: + - {{ .Release.Name }}-etcd-local + ip: 127.0.0.1 +{{- if gt (int .Values.replicas) 1 }} + serviceAccountName: {{ .Release.Name }}-etcd {{- end }} volumes: - name: etcd-config-file configMap: name: {{ .Release.Name }}-etcd-config - defaultMode: 0644 + defaultMode: 0640 items: - key: etcd.conf.yaml path: etcd.conf.yaml -{{- if .Values.etcdTLS }} - - name: ca-etcd +{{- if .Values.tls.etcd }} + - name: etcd-ca secret: secretName: {{ .Release.Name }}-etcd-ca - - name: etcd-tls + - name: etcd-server-tls + secret: + secretName: {{ .Release.Name }}-etcd-server-tls + - name: etcd-client-tls + secret: + secretName: {{ .Release.Name }}-etcd-client-tls + {{ if .Values.tls.etcd.peer }} + - name: etcd-peer-ca + secret: + secretName: {{ .Release.Name }}-etcd-peer-ca + - name: etcd-peer-server-tls secret: - secretName: {{ .Release.Name }}-etcd-tls + secretName: {{ .Release.Name }}-etcd-peer-server-tls + {{- end }} {{- end }} -{{- if .Values.backupRestoreTLS }} - - name: ca-etcdbr +{{- if .Values.tls.etcdBackupRestore }} + - name: backup-restore-ca secret: secretName: {{ .Release.Name }}-etcdbr-ca - - name: etcdbr-tls + - name: backup-restore-server-tls + secret: + secretName: {{ .Release.Name }}-etcdbr-server-tls + - name: backup-restore-client-tls secret: - secretName: {{ .Release.Name }}-etcdbr-tls + secretName: {{ .Release.Name }}-etcdbr-client-tls {{- end }} -{{- if and .Values.backup.storageProvider (not (eq .Values.backup.storageProvider "Local")) }} +{{- if .Values.backup.storageProvider }} + {{- if eq .Values.backup.storageProvider "Local"}} + - name: local-backup + hostPath: + path: {{ .Values.backup.local.path }}/{{ .Values.backup.storageContainer }} + type: Directory + {{- else }} - name: etcd-backup secret: secretName: {{ .Release.Name }}-etcd-backup + {{- end }} {{- end }} volumeClaimTemplates: - metadata: @@ -333,4 +366,3 @@ spec: resources: requests: storage: {{ .Values.storageCapacity }} - \ No newline at end of file diff --git a/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml b/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml index ca456fa05..290c9ae99 100644 --- a/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml @@ -1,4 +1,4 @@ -{{- if .Values.backupRestoreTLS }} +{{- if .Values.tls.etcdBackupRestore }} apiVersion: v1 kind: Secret metadata: @@ -10,5 +10,5 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} type: Opaque data: - ca.crt: {{ .Values.backupRestoreTLS.caBundle | b64enc }} -{{- end }} \ No newline at end of file + bundle.crt: {{ .Values.tls.etcdBackupRestore.ca | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/etcdbr-client-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcdbr-client-tls-secret.yaml new file mode 100644 index 000000000..c23d4350d --- /dev/null +++ b/chart/etcd-backup-restore/templates/etcdbr-client-tls-secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.tls.etcdBackupRestore }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-etcdbr-client-tls + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.tls.etcdBackupRestore.client.crt | b64enc }} + tls.key: {{ .Values.tls.etcdBackupRestore.client.key | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/etcdbr-server-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcdbr-server-tls-secret.yaml new file mode 100644 index 000000000..de8c03ef7 --- /dev/null +++ b/chart/etcd-backup-restore/templates/etcdbr-server-tls-secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.tls.etcdBackupRestore }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-etcdbr-server-tls + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.tls.etcdBackupRestore.server.crt | b64enc }} + tls.key: {{ .Values.tls.etcdBackupRestore.server.key | b64enc }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/role.yaml b/chart/etcd-backup-restore/templates/role.yaml new file mode 100644 index 000000000..9d80f6a48 --- /dev/null +++ b/chart/etcd-backup-restore/templates/role.yaml @@ -0,0 +1,18 @@ +{{- if gt (int .Values.replicas) 1 }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-etcd + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get +{{- end }} diff --git a/chart/etcd-backup-restore/templates/rolebinding.yaml b/chart/etcd-backup-restore/templates/rolebinding.yaml new file mode 100644 index 000000000..51c79b61a --- /dev/null +++ b/chart/etcd-backup-restore/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if gt (int .Values.replicas) 1 }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-etcd + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-etcd +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-etcd + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/chart/etcd-backup-restore/templates/serviceaccount.yaml b/chart/etcd-backup-restore/templates/serviceaccount.yaml new file mode 100644 index 000000000..7c2edd41b --- /dev/null +++ b/chart/etcd-backup-restore/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if gt (int .Values.replicas) 1 }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name}}-etcd + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/chart/etcd-backup-restore/values.yaml b/chart/etcd-backup-restore/values.yaml index b2221ef67..acfa4342b 100644 --- a/chart/etcd-backup-restore/values.yaml +++ b/chart/etcd-backup-restore/values.yaml @@ -1,14 +1,19 @@ images: + # Image to use for changing backup bucket permissions + changeBackupBucketPermissions: + repository: europe-docker.pkg.dev/gardener-project/public/3rd/alpine + tag: 3.20.3 + pullPolicy: IfNotPresent # etcd image to use - etcd: - # TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34 + etcdWrapper: repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcd-wrapper - tag: v0.2.0 + tag: latest pullPolicy: IfNotPresent # etcd-backup-restore image to use + # TODO: @anveshreddy18: use the latest tag for etcd-backup-restore once the v0.33.0 is released etcdBackupRestore: - repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcdbrctl - tag: v0.12.1 + repository: europe-docker.pkg.dev/gardener-project/snapshots/gardener/etcdbrctl + tag: v0.33.0-dev-e1690dd6ea14ca889d357307018ba2e53ced5203 pullPolicy: IfNotPresent resources: @@ -28,9 +33,13 @@ resources: memory: 128Mi servicePorts: - client: 2379 - server: 2380 - backupRestore: 8080 + etcd: + client: 2379 + peer: 2380 + etcdBackupRestore: + server: 8080 + etcdWrapper: + server: 9095 storageCapacity: 20Gi @@ -41,6 +50,8 @@ autoCompaction: mode: periodic retentionLength: "30m" +replicas: 1 + backup: # schedule is cron standard schedule to take full snapshots. schedule: "0 */1 * * *" @@ -60,7 +71,7 @@ backup: # garbageCollectionPeriod is the time period after which old snapshots are periodically garbage-collected garbageCollectionPeriod: "1m" - etcdConnectionTimeout: "30s" + etcdConnectionTimeout: "5m" etcdSnapshotTimeout: "8m" etcdDefragTimeout: "8m" # etcdQuotaBytes used to Raise alarms when backend DB size exceeds the given quota bytes @@ -68,27 +79,34 @@ backup: # storageContainer is name of the container or bucket name used for storage. # Directory name in case of local storage provider. - storageContainer: "/var/etcd/data/backup" + storageContainer: "etcd-bucket" # storageProvider indicate the type of backup storage provider. - # Supported values are ABS,GCS,S3,Swift,OSS,ECS,Local, empty means no backup. - storageProvider: "Local" + # Supported values are ABS,GCS,S3,Swift,OSS,OCS,ECS,Local, empty means no backup. + storageProvider: "" # compression defines the specification to compress the snapshots(full as well as delta). # it only supports 3 compression Policy: gzip(default), zlib, lzw. compression: enabled: true policy: "gzip" + leaderElection: + etcdConnectionTimeout: 5s + reelectionPeriod: 5s # failBelowRevision indicates the revision below which the validation of etcd will fail and restore will not be triggered in case # there is no snapshot on configured backup bucket. # failBelowRevision: 100000 # Please uncomment the following section based on the storage provider. + # local: + # path: "/etc/local-backupbuckets" # s3: # region: region-where-bucket-exists # secretAccessKey: secret-access-key-with-object-storage-privileges # accessKeyID: access-key-id-with-route53-privileges + # endpoint: endpoint-override-for-s3 # optional + # s3ForcePathStyle: "true" # optional # sseCustomerKey: aes-256-sse-customer-key # optional # sseCustomerAlgorithm: aes-256-sse-customer-algorithm # optional # gcs: @@ -98,7 +116,7 @@ backup: # abs: # storageAccount: storage-account-with-object-storage-privileges # storageKey: storage-key-with-object-storage-privileges - # domain: non-default-domain-for-object-storage-service + # domain: non-default-domain-for-object-storage-service # optional # emulatorEnabled: boolean-float-to-enable-e2e-tests-to-use-azure-emulator # optional # swift: # authURL: identity-server-url @@ -117,6 +135,11 @@ backup: # accessKeyID: access-key-id-with-object-storage-privileges # disableSsl: "false" # optional # insecureSkipVerify: "false" # optional + # ocs: + # accessKeyID: access-key-id-with-object-storage-privileges + # secretAccessKey: secret-access-key-with-object-storage-privileges + # endpoint: ocs-endpoint-url + # region: region-name # etcdAuth field contains the pre-created username-password pair # for etcd. Comment this whole section if you dont want to use @@ -125,35 +148,68 @@ etcdAuth: {} # username: username # password: password -etcdTLS: {} -# caBundle: | -# -----BEGIN CERTIFICATE----- -# ... -# -----END CERTIFICATE----- -# crt: | -# -----BEGIN CERTIFICATE----- -# ... -# -----END CERTIFICATE----- -# key: | -# -----BEGIN RSA PRIVATE KEY----- -# ... -# -----END RSA PRIVATE KEY----- - -# backupRestoreTLS field contains the pre-created secrets for backup-restore server. -# Comment this whole section if you dont want to use tls for the backup-restore server. -backupRestoreTLS: {} -# caBundle: | -# -----BEGIN CERTIFICATE----- -# ... -# -----END CERTIFICATE----- -# crt: | -# -----BEGIN CERTIFICATE----- -# ... -# -----END CERTIFICATE----- -# key: | -# -----BEGIN RSA PRIVATE KEY----- -# ... -# -----END RSA PRIVATE KEY----- - +tls: + etcd: {} + # ca: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # server: + # crt: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # key: | + # -----BEGIN PRIVATE KEY----- + # ... + # -----END PRIVATE KEY----- + # client: + # crt: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # key: | + # -----BEGIN PRIVATE KEY----- + # ... + # -----END PRIVATE KEY----- + # peer: + # ca: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # server: + # crt: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # key: | + # -----BEGIN PRIVATE KEY----- + # ... + # -----END PRIVATE KEY----- + # # etcdBackupRestore field contains the pre-created secrets for backup-restore server. + # # Comment this whole section if you dont want to use tls for the backup-restore server. + etcdBackupRestore: {} + # ca: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # server: + # crt: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # key: | + # -----BEGIN PRIVATE KEY----- + # ... + # -----END PRIVATE KEY----- + # client: + # crt: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # key: | + # -----BEGIN PRIVATE KEY----- + # ... + # -----END PRIVATE KEY----- # podAnnotations that will be passed to the resulting etcd pod podAnnotations: {} diff --git a/docs/deployment/getting_started.md b/docs/deployment/getting_started.md index 121cc0806..33b50f23d 100644 --- a/docs/deployment/getting_started.md +++ b/docs/deployment/getting_started.md @@ -164,6 +164,8 @@ INFO[0008] Successfully restored the etcd data directory. With sub-command `server` you can start a http server which exposes an endpoint to initialize etcd over REST interface. The server also keeps the backup schedule thread running to keep taking periodic backups. This is mainly made available to manage an etcd instance running in a Kubernetes cluster. You can deploy the example [helm chart](../../chart/etcd-backup-restore) on a Kubernetes cluster to have a fault-resilient, self-healing etcd cluster. +> **Note**: When deployed with the helm chart, only the static single member & static multi-member etcd cluster configurations are supported. The dynamic etcd cluster configuration is not supported. That is 0 to 1 or 0 to 3 member clusters are supported but not 1 to 3 member clusters. This is due to extra complexity in handling the scale-up scenario which cannot be brought into the helm charts at the moment. We recommend using [etcd-druid](https://github.com/gardener/etcd-druid/) for full-fledged etcd cluster management. + ## Etcdbrctl copy With sub-command `copy` you can copy all snapshots (Full and Delta) fom one snapstore to another. Using the two filter parameters `max-backups-to-copy` and `max-backup-age` you can also limit the number of snapshots that will be copied or target only the newest snapshots.