feat(helm): support registry image deletion and garbage-collect (#1029)

Because

- We are going support model image deletion

This commit

- add `maintenance` job to toggle `read-only` mode for `garbage-collect`
- fix `garbage-collect`

charts/core/templates/registry/configmap.yaml

@@ -7,7 +7,7 @@ metadata:
     {{- include "core.labels" . | nindent 4 }}
     app.kubernetes.io/component: registry
 data:
-  config.yaml: |+
+  config.yml: |+
     version: {{ .Values.registry.config.version }}
     log:
       {{- toYaml .Values.registry.config.log | nindent 6 }}
@@ -19,6 +19,14 @@ data:
       filesystem:
         {{- toYaml .Values.registry.config.storage.filesystem | nindent 8 }}
       {{- end }}
+      delete:
+        {{- toYaml .Values.registry.config.storage.delete | nindent 8 }}
+      redirect:
+        {{- toYaml .Values.registry.config.storage.redirect | nindent 8 }}
+      cache:
+        {{- toYaml .Values.registry.config.storage.cache | nindent 8 }}
+      maintenance:
+        {{- toYaml .Values.registry.config.storage.maintenance | nindent 8 }}
     http:
       {{- toYaml .Values.registry.config.http | nindent 6 }}
     redis:

charts/core/templates/registry/deployment.yaml

@@ -96,7 +96,7 @@ spec:
           volumeMounts:
             - name: config
               mountPath: {{ .Values.registry.configPath }}
-              subPath: config.yaml
+              subPath: config.yml
             {{- if eq .Values.registry.config.storage.type "filesystem" }}
             - name: data-volume
               mountPath: /var/lib/registry

charts/core/templates/registry/cronjob.yaml → charts/core/templates/registry/garbage-collect.yaml

@@ -1,5 +1,6 @@
 {{- if .Values.registry.enabled -}}
 {{- if .Values.registry.garbageCollect.enabled }}
+{{- $registry := .Values.persistence.persistentVolumeClaim.registry -}}
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -15,25 +16,23 @@ spec:
       labels:
         {{- include "core.matchLabels" . | nindent 8 }}
         app.kubernetes.io/component: registry
-      annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/registry/configmap.yaml") . | sha256sum }}
-        {{- with .Values.registry.podAnnotations }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
     spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 600
       template:
         spec:
+          securityContext:
+            runAsUser: 65534
+            runAsGroup: 65534
+            fsGroup: 65534
           {{- if .Values.registry.serviceAccountName }}
           serviceAccountName: {{ .Values.registry.serviceAccountName }}
           {{- end }}
+          automountServiceAccountToken: {{ .Values.registry.automountServiceAccountToken | default false }}
           {{- with .Values.imagePullSecrets }}
           imagePullSecrets:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          securityContext:
-            runAsUser: 65534
-            runAsGroup: 65534
-            fsGroup: 65534
           containers:
             - name: garbage-collect
               image: {{ .Values.registry.image.repository }}:{{ .Values.registry.image.tag }}
@@ -42,31 +41,53 @@ spec:
               - /bin/registry
               - garbage-collect
               - --delete-untagged={{ .Values.registry.garbageCollect.deleteUntagged }}
-              - /etc/docker/registry/config.yml
-              securityContext:
-                runAsUser: 65534
-                runAsGroup: 65534
-                fsGroup: 65534
+              - {{ .Values.registry.configPath }}
+              {{- if .Values.registry.resources }}
+              resources:
+                {{- toYaml .Values.registry.resources | nindent 16 }}
+              {{- end }}
               volumeMounts:
                 - name: config
                   mountPath: {{ .Values.registry.configPath }}
-                  subPath: config.yaml
+                  subPath: config.yml
+                {{- if eq .Values.registry.config.storage.type "filesystem" }}
+                - name: data-volume
+                  mountPath: /var/lib/registry
+                {{- end }}
+                {{- with .Values.registry.extraVolumeMounts }}
+                {{- toYaml . | nindent 16 }}
+                {{- end }}
           restartPolicy: OnFailure
           {{- with .Values.registry.nodeSelector }}
           nodeSelector:
-            {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
           {{- end }}
           {{- with .Values.registry.affinity }}
           affinity:
-            {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
           {{- end }}
           {{- with .Values.registry.tolerations }}
           tolerations:
-            {{- toYaml . | nindent 8 }}
+            {{- toYaml . | nindent 12 }}
           {{- end }}
           volumes:
             - name: config
               configMap:
                 name: {{ template "core.registry" . }}
+            {{- if eq .Values.registry.config.storage.type "filesystem" }}
+            - name: data-volume
+            {{- if not .Values.persistence.enabled }}
+              emptyDir: {}
+            {{- else if $registry.existingClaim }}
+              persistentVolumeClaim:
+                claimName: {{ $registry.existingClaim }}
+            {{- else }}
+              persistentVolumeClaim:
+                claimName: {{ template "core.registryDataVolume" . }}
+            {{- end }}
+            {{- end }}
+            {{- with .Values.registry.extraVolumes }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
 {{- end }}
 {{- end }}

charts/core/templates/registry/maintenance.yaml

@@ -0,0 +1,68 @@
+{{- if .Values.registry.enabled -}}
+{{- if .Values.registry.garbageCollect.enabled }}
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: {{ template "core.registry" . }}-maintenance
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "core.registry" . }}-maintenance
+rules:
+  - apiGroups: ["apps", "extensions"]
+    resources: ["deployments"]
+    verbs: ["get", "patch", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "core.registry" . }}-maintenance
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "core.registry" . }}-maintenance
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "core.registry" . }}-maintenance
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ template "core.registry" . }}-maintenance
+  labels:
+    {{- include "core.labels" . | nindent 4 }}
+    app.kubernetes.io/component: registry
+spec:
+  concurrencyPolicy: Forbid
+  schedule: {{ .Values.registry.garbageCollect.maintenanceSchedule | quote }}
+  jobTemplate:
+    metadata:
+      labels:
+        {{- include "core.matchLabels" . | nindent 8 }}
+        app.kubernetes.io/component: registry
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 600
+      template:
+        spec:
+          restartPolicy: Never
+          serviceAccountName: {{ template "core.registry" . }}-maintenance
+          containers:
+            - name: kubectl
+              image: bitnami/kubectl
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  TMP_CONFIGMAP_FILE="/tmp/configmap.yaml"
+                  kubectl get configmap {{ template "core.registry" . }} -n {{ template "core.namespace" . }} -o yaml > $TMP_CONFIGMAP_FILE
+                  sed -i '/readonly:/,/enabled:/s/enabled: true/enabled: TEMP/; /readonly:/,/enabled:/s/enabled: false/enabled: true/; /readonly:/,/enabled:/s/enabled: TEMP/enabled: false/' $TMP_CONFIGMAP_FILE
+                  kubectl apply -n {{ template "core.namespace" . }} -f $TMP_CONFIGMAP_FILE
+                  kubectl rollout restart -n {{ template "core.namespace" . }} deployment/{{ template "core.registry" . }} &&
+                  kubectl rollout status -n {{ template "core.namespace" . }} deployment/{{ template "core.registry" . }}
+{{- end }}
+{{- end }}

charts/core/values.yaml

@@ -1410,9 +1410,10 @@ registry:
       minAvailable:
       maxUnavailable:
   garbageCollect:
-    enabled: false
+    enabled: true
     deleteUntagged: true
-    schedule: "@midnight"
+    schedule: "10 0 * * 1"
+    maintenanceSchedule: "*/30 0 * * 1"
   config:
     version: 0.1
     log: