From 5b747a028078d5f59b581f6a6dd2099b4788feea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gaspard=20F=C3=A9rey?= Date: Wed, 15 Jan 2025 17:12:56 +0100 Subject: [PATCH] [Argilla] New Argilla Chart (#80) --- charts/argilla/Chart.yaml | 29 + charts/argilla/templates/NOTES.txt | 74 ++ charts/argilla/templates/_helpers.tpl | 10 + charts/argilla/templates/configmap.yaml | 8 + charts/argilla/templates/hpa.yaml | 23 + charts/argilla/templates/ingress.yaml | 34 + charts/argilla/templates/secret.yaml | 10 + charts/argilla/templates/statefulset.yaml | 167 ++++ charts/argilla/templates/svc.yaml | 15 + .../templates/test/test-connection.yaml | 17 + .../templates/test/test-python-api.yaml | 25 + charts/argilla/values.schema.json | 804 ++++++++++++++++++ charts/argilla/values.yaml | 151 ++++ 13 files changed, 1367 insertions(+) create mode 100644 charts/argilla/Chart.yaml create mode 100644 charts/argilla/templates/NOTES.txt create mode 100644 charts/argilla/templates/_helpers.tpl create mode 100644 charts/argilla/templates/configmap.yaml create mode 100644 charts/argilla/templates/hpa.yaml create mode 100644 charts/argilla/templates/ingress.yaml create mode 100644 charts/argilla/templates/secret.yaml create mode 100644 charts/argilla/templates/statefulset.yaml create mode 100644 charts/argilla/templates/svc.yaml create mode 100644 charts/argilla/templates/test/test-connection.yaml create mode 100644 charts/argilla/templates/test/test-python-api.yaml create mode 100644 charts/argilla/values.schema.json create mode 100644 charts/argilla/values.yaml diff --git a/charts/argilla/Chart.yaml b/charts/argilla/Chart.yaml new file mode 100644 index 0000000..9a1f734 --- /dev/null +++ b/charts/argilla/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: argilla +description: Argilla is a data annotation and curation platform designed to streamline the process of creating, managing, and labeling datasets for machine learning and natural language processing (NLP) tasks +icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argilla.svg +keywords: + - Data Annotation + - Data Curation + - Machine Learning + - NLP +home: https://argilla.io/ +sources: + - https://github.com/InseeFrLab/helm-charts-miscellaneous/tree/master/charts/argilla + - https://github.com/argilla-io/argilla/ +type: application +version: 1.0.0 +appVersion: "2.3.0" +dependencies: +- name: redis + version: 20.2.0 + repository: https://charts.bitnami.com/bitnami +- name: postgresql + version: 16.0.1 + repository: https://charts.bitnami.com/bitnami +- name: elasticsearch + version: 21.3.21 + repository: https://charts.bitnami.com/bitnami +- name: library-chart + version: 1.5.25 + repository: https://inseefrlab.github.io/helm-charts-interactive-services diff --git a/charts/argilla/templates/NOTES.txt b/charts/argilla/templates/NOTES.txt new file mode 100644 index 0000000..e03b8e9 --- /dev/null +++ b/charts/argilla/templates/NOTES.txt @@ -0,0 +1,74 @@ +{{- $pvcList := list -}} +{{- if .Values.argilla.persistence.enabled -}} +{{- $pvcList = append $pvcList (printf "data-%s-0" (include "library-chart.fullname" .)) -}} +{{- end -}} +{{- if .Values.redis.master.persistence.enabled -}} +{{- $pvcList = append $pvcList (printf "redis-data-%s-master-0" (include "common.names.fullname" .Subcharts.redis)) -}} +{{- end -}} +{{- if .Values.postgresql.primary.persistence.enabled -}} +{{- $pvcList = append $pvcList (printf "%s-%s-0" (.Values.postgresql.primary.persistence.volumeName) (include "postgresql.v1.primary.fullname" .Subcharts.postgresql)) -}} +{{- end -}} +{{- if .Values.elasticsearch.master.persistence.enabled -}} +{{- $pvcList = append $pvcList (printf "data-%s-0" (include "elasticsearch.master.fullname" .Subcharts.elasticsearch)) -}} +{{- end -}} +{{- if eq .Values.userPreferences.language "fr" -}} +- Votre nom d'utilisateur est **{{ .Values.argilla.auth.username }}** +- Votre mot de passe est **{{ .Values.argilla.auth.password }}** +- Votre clef API est **{{ .Values.argilla.auth.apikey }}** + +{{- if .Values.ingress.enabled }} +- Vous pouvez vous connecter à Argilla avec votre navigateur à ce [lien](http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .Values.ingress.hostname }}) +{{- end }} +{{- if .Values.route.enabled }} +- Vous pouvez vous connecter à Argilla avec votre navigateur à ce [lien](https://{{ .Values.route.hostname }}) +{{- end }} +- Vous pouvez vous connectez à Argilla depuis l'intérieur du datalab à cette URL : **http://{{ include "library-chart.fullname" . }}:6900** +- Example using the Python client API : + +``` +import argilla as rg +client = rg.Argilla(api_url="http://{{ include "library-chart.fullname" . }}:6900", api_key="{{ .Values.argilla.auth.apikey }}") +rg.Workspace(name="my_workspace", client=client).create() +``` +{{- if not (empty $pvcList) }} + +**NOTES sur la suppression :** + +- **Vous pouvez supprimer ce chart en toute sécurité et en recréer un plus tard** +- Les volumes de données ne seront pas supprimés +- Si vous démarrez un nouveau {{ .Chart.Name }}, il réutilisera automatiquement ces volumes. +- Si vous souhaitez supprimer définitivement ces volumes : `kubectl delete pvc {{ join " " $pvcList }}` +{{- end -}} +{{- else -}} + {{- if ne .Values.userPreferences.language "en" -}} + *NOTES are not supported for language `{{ .Values.userPreferences.language }}`... Defaulting to english.* + + {{ end -}} +- Your username is **{{ .Values.argilla.auth.username }}** +- Your password is **{{ .Values.argilla.auth.password }}** +- Your API key is **{{ .Values.argilla.auth.apikey }}** + +{{- if .Values.ingress.enabled }} +- You can connect to Argilla with your browser on this [link](http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .Values.ingress.hostname }}) +{{- end }} +{{- if .Values.route.enabled }} +- You can connect to Argilla with your browser on this [link](https://{{ .Values.route.hostname }}) +{{- end }} +- You can connect to Argilla from inside the datalab at this url : **http://{{ include "library-chart.fullname" . }}:6900** +- Example using the Python client API : + +``` +import argilla as rg +client = rg.Argilla(api_url="http://{{ include "library-chart.fullname" . }}:6900", api_key="{{ .Values.argilla.auth.apikey }}") +rg.Workspace(name="my_workspace", client=client).create() +``` +{{- if not (empty $pvcList) }} + +*NOTES about deletion:* + +- **You can safely delete this chart and recreate one later** +- The underlying data volumes will not be deleted +- If you start a new Argilla service, it will automatically reuse those volumes +- If you want to delete those volumes definitely: `kubectl delete pvc {{ join " " $pvcList }}` +{{- end -}} +{{- end -}} diff --git a/charts/argilla/templates/_helpers.tpl b/charts/argilla/templates/_helpers.tpl new file mode 100644 index 0000000..094b18b --- /dev/null +++ b/charts/argilla/templates/_helpers.tpl @@ -0,0 +1,10 @@ + +{{/* Returns a connexion string to the Postgres database. */}} +{{- define "argilla.postgresql.connexionstring" -}} +{{- $username := include "postgresql.v1.username" .Subcharts.postgresql -}} +{{- $password := .Values.global.postgresql.auth.password -}} +{{- $database := include "postgresql.v1.database" .Subcharts.postgresql -}} +{{- $svcname := include "postgresql.v1.primary.fullname" .Subcharts.postgresql -}} +{{- $svcport := include "postgresql.v1.service.port" .Subcharts.postgresql -}} +{{- printf "postgresql+asyncpg://%s:%s@%s:%s/%s" $username $password $svcname $svcport $database -}} +{{- end -}} diff --git a/charts/argilla/templates/configmap.yaml b/charts/argilla/templates/configmap.yaml new file mode 100644 index 0000000..4a970dc --- /dev/null +++ b/charts/argilla/templates/configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hosts-file +data: + hosts: | + 127.0.0.1 localhost + ::1 localhost diff --git a/charts/argilla/templates/hpa.yaml b/charts/argilla/templates/hpa.yaml new file mode 100644 index 0000000..a5ad893 --- /dev/null +++ b/charts/argilla/templates/hpa.yaml @@ -0,0 +1,23 @@ +{{- if .Values.argilla.hpa.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "library-chart.fullname" . }} + labels: + {{- include "library-chart.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "library-chart.fullname" . }} + minReplicas: {{ .Values.replicas.min }} + maxReplicas: {{ .Values.replicas.max }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 +{{ end }} \ No newline at end of file diff --git a/charts/argilla/templates/ingress.yaml b/charts/argilla/templates/ingress.yaml new file mode 100644 index 0000000..f9bffc9 --- /dev/null +++ b/charts/argilla/templates/ingress.yaml @@ -0,0 +1,34 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "library-chart.fullname" . }} + labels: + {{- include "library-chart.labels" . | nindent 4 }} + app.kubernetes.io/component: server + annotations: + {{- include "library-chart.ingress.annotations" . | nindent 4 }} +spec: + {{- if and .Values.ingress.ingressClassName (eq "true" (include "library-chart.ingress.supportsIngressClassname" .)) }} + ingressClassName: {{ .Values.ingress.ingressClassName | quote }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + - hosts: + - {{ .Values.ingress.hostname | quote }} + {{- if .Values.ingress.useCertManager }} + secretName: tls-cert-{{ include "library-chart.fullname" . }} + {{- end }} + {{- end }} + rules: + - host: {{ .Values.ingress.hostname | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "library-chart.fullname" . }} + port: + number: 6900 +{{- end }} \ No newline at end of file diff --git a/charts/argilla/templates/secret.yaml b/charts/argilla/templates/secret.yaml new file mode 100644 index 0000000..704ac47 --- /dev/null +++ b/charts/argilla/templates/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "library-chart.fullname" . }} +type: Opaque +data: + username: {{ .Values.argilla.auth.username | b64enc | quote }} + password: {{ .Values.argilla.auth.password | b64enc | quote }} + apikey: {{ .Values.argilla.auth.apikey | b64enc | quote }} + secretkey: {{ .Values.argilla.auth.secretkey | b64enc | quote }} diff --git a/charts/argilla/templates/statefulset.yaml b/charts/argilla/templates/statefulset.yaml new file mode 100644 index 0000000..8b90761 --- /dev/null +++ b/charts/argilla/templates/statefulset.yaml @@ -0,0 +1,167 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "library-chart.fullname" . }} + labels: + {{- include "library-chart.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + replicas: {{ .Values.argilla.replicaCount }} + selector: + matchLabels: + {{- include "library-chart.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + template: + metadata: + labels: + {{- include "library-chart.labels" . | nindent 8 }} + {{ template "postgresql.v1.primary.fullname" .Subcharts.postgresql }}-client: "true" + {{ template "common.names.fullname" .Subcharts.elasticsearch }}-client: "true" + {{ template "common.names.fullname" .Subcharts.redis }}-client: "true" + app.kubernetes.io/component: server + spec: + initContainers: + - name: permissions-fixer + image: alpine + securityContext: + runAsUser: 0 + volumeMounts: + - name: data + mountPath: /data + command: [ "sh", "-c"] + args: + - | + chown -R 1000:1000 /data + chmod -R a+w /data + resources: + limits: + cpu: 50m + memory: 50Mi + - name: wait-for-postgresql + image: alpine/curl:latest + imagePullPolicy: IfNotPresent + command: [ "sh", "-c"] + args: + - | + until echo PING | nc $POSTGRES_HOST $POSTGRES_PORT + do + echo Postgres not ready. Sleeping... + sleep 5 + done + echo Postgres up! + env: + - name: POSTGRES_HOST + value: {{ include "postgresql.v1.primary.fullname" .Subcharts.postgresql | quote }} + - name: POSTGRES_PORT + value: {{ include "postgresql.v1.service.port" .Subcharts.postgresql | quote }} + resources: + limits: + cpu: 50m + memory: 50Mi + - name: wait-for-redis + image: alpine/curl:latest + imagePullPolicy: IfNotPresent + command: [ "sh", "-c"] + args: + - | + until echo PING | nc $REDIS_HOST 6379 + do + echo Redis not ready. Sleeping... + sleep 5 + done + echo Redis up! + env: + - name: REDIS_HOST + value: {{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) | quote }} + - name: wait-for-elasticsearch + image: alpine/curl:latest + imagePullPolicy: IfNotPresent + command: [ "sh", "-c"] + args: + - | + status_code=$(curl -s -o /dev/null -w '%{http_code}' $ELASTICSEARCH_URL) + while [ $status_code -ne 200 ] + do + echo Elasticsearch not ready. Sleeping... + sleep 5 + status_code=$(curl -s -o /dev/null -w '%{http_code}' $ELASTICSEARCH_URL) + done + echo Elasticsearch up! + env: + - name: ELASTICSEARCH_URL + value: "http://{{ include "elasticsearch.service.name" .Subcharts.elasticsearch }}:9200" + resources: + limits: + cpu: 50m + memory: 50Mi + containers: + - name: argilla-server + image: {{ printf "%s/%s:%s" .Values.argilla.image.registry .Values.argilla.image.repository .Values.argilla.image.tag }} + imagePullPolicy: {{ .Values.argilla.image.pullPolicy | quote }} + env: + - name: USERNAME + valueFrom: + secretKeyRef: + name: {{ include "library-chart.fullname" . }} + key: username + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "library-chart.fullname" . }} + key: password + - name: API_KEY + valueFrom: + secretKeyRef: + name: {{ include "library-chart.fullname" . }} + key: apikey + - name: ARGILLA_AUTH_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "library-chart.fullname" . }} + key: secretkey + - name: ARGILLA_HOME_PATH + value: "/data" + - name: ARGILLA_ELASTICSEARCH + value: {{ printf "http://%s:9200" (include "elasticsearch.service.name" .Subcharts.elasticsearch) | quote }} + - name: ARGILLA_REDIS_URL + value: {{ printf "redis://%s-master:6379/0" (include "common.names.fullname" .Subcharts.redis) | quote }} + - name: ARGILLA_DATABASE_URL + value: {{ include "argilla.postgresql.connexionstring" . | quote }} + ports: + - containerPort: 6900 + volumeMounts: + - name: hosts-file + mountPath: /etc/hosts + subPath: hosts + - name: data + mountPath: /data + resources: + {{- toYaml .Values.argilla.resources | nindent 12 }} + volumes: + - name: hosts-file + configMap: + name: hosts-file + - name: data +{{- if not .Values.argilla.persistence.enabled }} + emptyDir: + sizeLimit: {{ .Values.argilla.persistence.size | quote }} +{{- else }} + persistentVolumeClaim: + claimName: data-{{ include "library-chart.fullname" . }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: {{ .Values.argilla.persistence.size | quote }} + {{- if .Values.global.defaultStorageClass }} + {{- if (eq "-" .Values.global.defaultStorageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.global.defaultStorageClass }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/argilla/templates/svc.yaml b/charts/argilla/templates/svc.yaml new file mode 100644 index 0000000..ae2545d --- /dev/null +++ b/charts/argilla/templates/svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "library-chart.fullname" . }} + labels: + {{- include "library-chart.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + type: ClusterIP + selector: + {{- include "library-chart.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server + ports: + - name: http + port: 6900 diff --git a/charts/argilla/templates/test/test-connection.yaml b/charts/argilla/templates/test/test-connection.yaml new file mode 100644 index 0000000..8ac4b76 --- /dev/null +++ b/charts/argilla/templates/test/test-connection.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "library-chart.fullname" . }}-test-connection" + labels: + {{- include "library-chart.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": hook-succeeded +spec: + containers: + - name: test-connection + image: busybox + command: ['wget'] + args: ['http://{{ include "library-chart.fullname" . }}:6900'] + restartPolicy: Never diff --git a/charts/argilla/templates/test/test-python-api.yaml b/charts/argilla/templates/test/test-python-api.yaml new file mode 100644 index 0000000..6a8dc77 --- /dev/null +++ b/charts/argilla/templates/test/test-python-api.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "library-chart.fullname" . }}-test-python-client" + labels: + {{- include "library-chart.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": hook-succeeded +spec: + containers: + - name: test-python-client + image: "inseefrlab/onyxia-python-minimal" + command: ["/bin/sh","-c"] + args: + - | + pip3 install argilla && timeout 2m python3 <