fix: Add tests for operator-wandb (#338)

wandb · Feb 13, 2025 · ab61675 · ab61675
1 parent 884d184
commit ab61675
Show file tree

Hide file tree

Showing 16 changed files with 521 additions and 32 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -0,0 +1,43 @@
+name: Lint Charts
+
+on:
+  pull_request:
+    paths: charts/**
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    environment: Helm Charts
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3.4
+        with:
+          version: v3.16.3
+
+      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+      # yamllint (https://github.com/adrienverge/yamllint) which require Python
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.6.1
+        with:
+          version: v3.11.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config ct.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing  (lint)
+        run: ct lint --config ct.yaml
diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
@@ -0,0 +1,71 @@
+name: Test operator-wandb Chart
+
+on:
+  pull_request:
+    paths: charts/operator-wandb/**
+
+jobs:
+  test:
+    name: Test Chart
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
+        configuration: ["default"]
+    runs-on: ubuntu-latest
+    environment: Helm Charts
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3.4
+        with:
+          version: v3.16.3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.6.1
+        with:
+          version: v3.11.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config ct.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.12.0
+        with:
+          version: v0.26.0
+          cluster_name: chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}
+          node_image: kindest/node:${{ matrix.k8s-version }}
+        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+
+      - name: Install Minio
+        run: kubectl --context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} apply -f test-configs/minio/default.yaml
+        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+
+      - name: Wait for Minio to be Ready
+        run: kubectl --context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} -n minio wait --for=condition=Ready pod/minio --timeout=300s
+        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        env:
+          LICENSE: ${{ secrets.LICENSE }}
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          ct install \
+          --charts ./charts/operator-wandb \
+          --config ct.yaml \
+          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --timeout 600s' \
+          --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'
diff --git a/.github/workflows/test-operator.yaml b/.github/workflows/test-operator.yaml
@@ -0,0 +1,54 @@
+# TODO(dpanzella): Uncomment this file when the ready for automated tests of the operator chart
+
+#name: Test operator Chart
+#
+#on:
+#  pull_request:
+#    paths: charts/operator/**
+#
+#jobs:
+#  test:
+#    runs-on: ubuntu-latest
+#    environment: Helm Charts
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v2
+#        with:
+#          fetch-depth: 0
+#
+#      - name: Set up Helm
+#        uses: azure/setup-helm@v3.4
+#        with:
+#          version: v3.16.3
+#
+#      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+#      # yamllint (https://github.com/adrienverge/yamllint) which require Python
+#      - name: Set up Python
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: "3.10"
+#
+#      - name: Set up chart-testing
+#        uses: helm/chart-testing-action@v2.6.1
+#        with:
+#          version: v3.11.0
+#
+#      - name: Run chart-testing (list-changed)
+#        id: list-changed
+#        run: |
+#          changed=$(ct list-changed --config ct.yaml)
+#          if [[ -n "$changed" ]]; then
+#            echo "::set-output name=changed::true"
+#          fi
+#
+#      - name: Create kind cluster
+#        uses: helm/kind-action@v1.8.0
+#        with:
+#          version: v0.20.0
+#        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+#
+#      - name: Run chart-testing (install)
+#        env:
+#          LICENSE: ${{ secrets.LICENSE }}
+#        if: steps.list-changed.outputs.changed == 'true'
+#        run: ct install --charts ./charts/operator --config ct.yaml --helm-extra-set-args --set=license=$LICENSE
diff --git a/.github/workflows/lint-test.yaml → .github/workflows/test-wandb.yaml b/.github/workflows/lint-test.yaml → .github/workflows/test-wandb.yaml
@@ -1,9 +1,11 @@
-name: Lint and Test Charts
+name: Test wandb Chart
 
-on: pull_request
+on:
+  pull_request:
+    paths: charts/wandb/**
 
 jobs:
-  lint-test:
+  test:
     runs-on: ubuntu-latest
     environment: Helm Charts
     steps:
@@ -37,9 +39,6 @@ jobs:
             echo "::set-output name=changed::true"
           fi
 
-      - name: Run chart-testing  (lint)
-        run: ct lint --config ct.yaml
-
       - name: Create kind cluster
         uses: helm/kind-action@v1.8.0
         with:

diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.26.2
+version: 0.26.3
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

diff --git a/charts/operator-wandb/charts/bufstream/config/bufstream.yaml.tpl b/charts/operator-wandb/charts/bufstream/config/bufstream.yaml.tpl
@@ -24,7 +24,7 @@ storage:
   {{- end }}
   {{- if .Values.storage.s3.accessKeyId }}
   access_key_id:
-    string: {{ .Values.storage.s3.accessKeyId | quote }}
+    string: {{ tpl .Values.storage.s3.accessKeyId . | quote }}
   secret_access_key:
     path: /config/secrets/storage/secret_access_key
   {{- end }}
@@ -40,7 +40,7 @@ storage:
 etcd:
   addresses:
 {{- range .Values.metadata.etcd.addresses }}
-  - host: {{ .host | quote }}
+  - host: {{ tpl .host $ | quote }}
     port: {{ .port }}
 {{- end }}
 {{- with .Values.metadata.etcd.tls }}

diff --git a/charts/operator-wandb/charts/bufstream/templates/deployment.yaml b/charts/operator-wandb/charts/bufstream/templates/deployment.yaml
@@ -248,9 +248,9 @@ spec:
         {{- if and (eq .Values.storage.use "s3") (.Values.storage.s3.accessKeyId) }}
         - name: storage
           secret:
-            secretName: {{ .Values.storage.s3.secretName }}
+            secretName: {{ tpl .Values.storage.s3.secretName . }}
             items:
-              - key: secret_access_key
+              - key: {{ tpl .Values.storage.s3.secretKeyName . }}
                 path: secret_access_key
         {{- end }}
         {{- if and (eq .Values.storage.use "gcs") (.Values.storage.gcs.secretName) }}

diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -61,17 +61,17 @@ spec:
             {{- include "mysql.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
           livenessProbe:
-            exec:
-              command:
-              - sh
-              - -c
-              - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
+            tcpSocket:
+              port: 3306
           readinessProbe:
-            exec:
-              command:
-              - sh
-              - -c
-              - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
+            tcpSocket:
+              port: 3306
+          startupProbe:
+            initialDelaySeconds: 20
+            periodSeconds: 5
+            failureThreshold: 60
+            tcpSocket:
+              port: 3306
           volumeMounts:
             - name: data
               mountPath: /var/lib/mysql

diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml
@@ -29,9 +29,9 @@ pod:
   annotations: {}
   securityContext:
     runAsNonRoot: true
-    runAsUser: 1000
+    runAsUser: 999
     runAsGroup: 0
-    fsGroup: 1000
+    fsGroup: 999
     fsGroupChangePolicy: "OnRootMismatch"
     seccompProfile:
       type: ""

diff --git a/charts/operator-wandb/templates/redis.yaml b/charts/operator-wandb/templates/redis.yaml
@@ -1,16 +1,14 @@
-{{- if not .Values.redis.install }}
-{{- $secretName := (include "wandb.redis.passwordSecret" .)  }}
+{{- if not .Values.global.redis.secret.secretName }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: "{{ $secretName }}"
+  name: "{{ include "wandb.redis.passwordSecret" . }}"
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 data:
   REDIS_PASSWORD: {{ include "wandb.redis.password" . | b64enc }}
   REDIS_CA_CERT: {{ include "wandb.redis.caCert" . | b64enc }}
 {{- end }}
-
 ---
 apiVersion: v1
 kind: ConfigMap

diff --git a/charts/operator-wandb/templates/tests/test-connection.yaml b/charts/operator-wandb/templates/tests/test-connection.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "wandb.fullname" . }}-test-connection"
+  labels:
+    {{- include "wandb.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wandb-verify
+      image: python:3.10
+      env:
+        - name: WANDB_BASE_URL
+          value: "http://{{ .Release.Name }}-app:8080"
+        - name: WANDB_API_KEY
+          value: "test-api-key"
+      # wandb verify is terribly flaky, there's a PR up to fix it until then we try 3 times
+      command:
+        - sh
+        - -c
+        - "pip install wandb==0.17.8 && ((sleep 30 && wandb verify) || (sleep 30 && wandb verify) || (sleep 30 && wandb verify))"
+  restartPolicy: Never
diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml
@@ -336,7 +336,9 @@ redis:
   nameOverride: "redis"
   architecture: standalone
   auth:
-    enabled: false
+    enabled: true
+    existingSecret: '{{ include "wandb.redis.passwordSecret" . }}'
+    existingSecretPasswordKey: '{{ .Values.global.redis.secret.secretKey }}'
   metrics:
     enabled: false
     service:
@@ -539,7 +541,7 @@ glue:
     "{{ .Release.Name }}-mysql": "secretRef"
     "{{ .Release.Name }}-bucket-configmap": "configMapRef"
     "{{ .Release.Name }}-mysql-configmap": "configMapRef"
-    "{{ .Release.Name }}-redis-secret": "secretRef"
+    '{{ include "wandb.redis.passwordSecret" . }}': "secretRef"
     "{{ .Release.Name }}-redis-configmap": "configMapRef"
     "{{ .Release.Name }}-global-secret": "secretRef"
     "{{ .Release.Name }}-gorilla-secret": "secretRef"
@@ -635,7 +637,7 @@ glue:
         optional: true
     - name: redis-ca
       secret:
-        secretName: "{{ .Release.Name }}-redis-secret"
+        secretName: '{{ include "wandb.redis.passwordSecret" . }}'
         items:
           - key: REDIS_CA_CERT
             path: redis_ca.pem
@@ -730,7 +732,7 @@ api:
     "{{ .Release.Name }}-mysql": "secretRef"
     "{{ .Release.Name }}-bucket-configmap": "configMapRef"
     "{{ .Release.Name }}-mysql-configmap": "configMapRef"
-    "{{ .Release.Name }}-redis-secret": "secretRef"
+    '{{ include "wandb.redis.passwordSecret" . }}': "secretRef"
     "{{ .Release.Name }}-redis-configmap": "configMapRef"
     "{{ .Release.Name }}-kafka-configmap": "configMapRef"
     "{{ .Release.Name }}-global-secret": "secretRef"
@@ -832,7 +834,7 @@ api:
         optional: true
     - name: redis-ca
       secret:
-        secretName: "{{ .Release.Name }}-redis-secret"
+        secretName: '{{ include "wandb.redis.passwordSecret" . }}'
         items:
           - key: REDIS_CA_CERT
             path: redis_ca.pem